35 #include "llvm/ADT/APFloat.h"
36 #include "llvm/ADT/APInt.h"
37 #include "llvm/ADT/FloatingPointMode.h"
38 #include "llvm/ADT/SmallPtrSet.h"
39 #include "llvm/ADT/StringExtras.h"
40 #include "llvm/Analysis/ValueTracking.h"
41 #include "llvm/IR/DataLayout.h"
42 #include "llvm/IR/FPAccuracy.h"
43 #include "llvm/IR/InlineAsm.h"
44 #include "llvm/IR/Intrinsics.h"
45 #include "llvm/IR/IntrinsicsAArch64.h"
46 #include "llvm/IR/IntrinsicsAMDGPU.h"
47 #include "llvm/IR/IntrinsicsARM.h"
48 #include "llvm/IR/IntrinsicsBPF.h"
49 #include "llvm/IR/IntrinsicsDirectX.h"
50 #include "llvm/IR/IntrinsicsHexagon.h"
51 #include "llvm/IR/IntrinsicsNVPTX.h"
52 #include "llvm/IR/IntrinsicsPowerPC.h"
53 #include "llvm/IR/IntrinsicsR600.h"
54 #include "llvm/IR/IntrinsicsRISCV.h"
55 #include "llvm/IR/IntrinsicsS390.h"
56 #include "llvm/IR/IntrinsicsVE.h"
57 #include "llvm/IR/IntrinsicsWebAssembly.h"
58 #include "llvm/IR/IntrinsicsX86.h"
59 #include "llvm/IR/MDBuilder.h"
60 #include "llvm/IR/MatrixBuilder.h"
61 #include "llvm/Support/ConvertUTF.h"
62 #include "llvm/Support/MathExtras.h"
63 #include "llvm/Support/ScopedPrinter.h"
64 #include "llvm/TargetParser/AArch64TargetParser.h"
65 #include "llvm/TargetParser/X86TargetParser.h"
69 using namespace clang;
70 using namespace CodeGen;
74 Align AlignmentInBytes) {
76 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
81 Byte = CGF.
Builder.getInt8(0x00);
85 Byte = llvm::dyn_cast<llvm::ConstantInt>(
93 I->addAnnotationMetadata(
"auto-init");
108 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
109 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
110 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
111 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
112 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
113 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
114 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
115 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
116 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
117 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
118 {Builtin::BI__builtin_printf,
"__printfieee128"},
119 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
120 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
121 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
122 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
123 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
124 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
125 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
126 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
127 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
128 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
129 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
130 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
131 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
137 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
138 {Builtin::BI__builtin_frexpl,
"frexp"},
139 {Builtin::BI__builtin_ldexpl,
"ldexp"},
140 {Builtin::BI__builtin_modfl,
"modf"},
146 if (FD->
hasAttr<AsmLabelAttr>())
147 Name = getMangledName(D);
151 if (getTriple().isPPC64() &&
152 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
153 F128Builtins.contains(BuiltinID))
154 Name = F128Builtins[BuiltinID];
155 else if (getTriple().isOSAIX() &&
156 &getTarget().getLongDoubleFormat() ==
157 &llvm::APFloat::IEEEdouble() &&
158 AIXLongDouble64Builtins.contains(BuiltinID))
159 Name = AIXLongDouble64Builtins[BuiltinID];
164 llvm::FunctionType *Ty =
165 cast<llvm::FunctionType>(getTypes().ConvertType(FD->
getType()));
167 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
173 QualType T, llvm::IntegerType *IntType) {
176 if (
V->getType()->isPointerTy())
177 return CGF.
Builder.CreatePtrToInt(
V, IntType);
179 assert(
V->getType() == IntType);
187 if (ResultType->isPointerTy())
188 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
190 assert(
V->getType() == ResultType);
201 if (Align % Bytes != 0) {
214 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
224 llvm::IntegerType *IntType = llvm::IntegerType::get(
228 llvm::Type *ValueType = Val->getType();
231 llvm::Value *Result =
256 llvm::AtomicRMWInst::BinOp
Kind,
265 llvm::AtomicRMWInst::BinOp
Kind,
267 Instruction::BinaryOps Op,
268 bool Invert =
false) {
277 llvm::IntegerType *IntType = llvm::IntegerType::get(
281 llvm::Type *ValueType = Val->getType();
285 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
286 Result = CGF.
Builder.CreateBinOp(Op, Result, Val);
289 CGF.
Builder.CreateBinOp(llvm::Instruction::Xor, Result,
290 llvm::ConstantInt::getAllOnesValue(IntType));
314 llvm::IntegerType *IntType = llvm::IntegerType::get(
318 llvm::Type *ValueType = Cmp->getType();
323 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
324 llvm::AtomicOrdering::SequentiallyConsistent);
327 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
350 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
365 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
366 AtomicOrdering::Monotonic :
374 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
375 Result->setVolatile(
true);
376 return CGF.
Builder.CreateExtractValue(Result, 0);
393 AtomicOrdering SuccessOrdering) {
400 assert(DestPtr->getType()->isPointerTy());
401 assert(!ExchangeHigh->getType()->isPointerTy());
402 assert(!ExchangeLow->getType()->isPointerTy());
405 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
406 ? AtomicOrdering::Monotonic
411 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
412 Address DestAddr(DestPtr, Int128Ty,
417 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
418 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
420 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
421 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
427 SuccessOrdering, FailureOrdering);
433 CXI->setVolatile(
true);
440 Value *Success = CGF.
Builder.CreateExtractValue(CXI, 1);
445 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
452 return CGF.
Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
457 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
464 return CGF.
Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
475 Load->setVolatile(
true);
485 llvm::StoreInst *
Store =
487 Store->setVolatile(
true);
492 llvm::Function *FPBuiltinF,
495 llvm::CallInst *CI = CGF.
Builder.CreateCall(FPBuiltinF, Args);
498 llvm::AttributeList AttrList;
500 llvm::Metadata *AspectMD =
nullptr;
504 Name, AttrList, AspectMD,
ID,
505 Name ==
"sincos" ? Args[0]->getType() : FPBuiltinF->getReturnType());
506 CI->setAttributes(AttrList);
509 CI->setMetadata(
"sycl_used_aspects",
515 unsigned FPIntrinsicID,
unsigned IntrinsicID,
516 bool HasAccuracyRequirement) {
517 return HasAccuracyRequirement
530 unsigned FPAccuracyIntrinsicID,
531 unsigned IntrinsicID, llvm::Value *Src0,
534 if (FPAccuracyIntrinsicID != Intrinsic::not_intrinsic) {
559 unsigned ConstrainedIntrinsicID,
560 unsigned FPAccuracyIntrinsicID = Intrinsic::not_intrinsic) {
564 IntrinsicID, Src0, Name);
567 FPAccuracyIntrinsicID);
570 if (CGF.
Builder.getIsFPConstrained()) {
572 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
575 return CGF.
Builder.CreateCall(F, Src0);
583 unsigned ConstrainedIntrinsicID,
584 unsigned FPAccuracyIntrinsicID = Intrinsic::not_intrinsic) {
589 IntrinsicID, Src0, Name);
592 FPAccuracyIntrinsicID);
593 if (CGF.
Builder.getIsFPConstrained()) {
596 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
599 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
610 if (CGF.
Builder.getIsFPConstrained()) {
613 {Src0->getType(), Src1->getType()});
614 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
619 return CGF.
Builder.CreateCall(F, {Src0, Src1});
625 const CallExpr *E,
unsigned IntrinsicID,
626 unsigned ConstrainedIntrinsicID) {
631 if (CGF.
Builder.getIsFPConstrained()) {
634 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
637 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
644 unsigned IntrinsicID,
645 unsigned ConstrainedIntrinsicID,
649 if (CGF.
Builder.getIsFPConstrained())
654 if (CGF.
Builder.getIsFPConstrained())
655 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
657 return CGF.
Builder.CreateCall(F, Args);
663 unsigned IntrinsicID,
664 llvm::StringRef Name =
"") {
668 return CGF.
Builder.CreateCall(F, Src0, Name);
674 unsigned IntrinsicID) {
679 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
685 unsigned IntrinsicID) {
691 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
697 unsigned IntrinsicID) {
702 return CGF.
Builder.CreateCall(F, {Src0, Src1});
708 unsigned IntrinsicID,
709 unsigned ConstrainedIntrinsicID) {
713 if (CGF.
Builder.getIsFPConstrained()) {
716 {ResultType, Src0->getType()});
717 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
721 return CGF.
Builder.CreateCall(F, Src0);
734 llvm::Value *Call = CGF.
Builder.CreateCall(F, Src0);
736 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(Call, 1);
740 return CGF.
Builder.CreateExtractValue(Call, 0);
746 llvm::CallInst *Call = CGF.
Builder.CreateCall(F,
V);
747 Call->setDoesNotAccessMemory();
756 llvm::Type *Ty =
V->getType();
757 int Width = Ty->getPrimitiveSizeInBits();
758 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
760 if (Ty->isPPC_FP128Ty()) {
770 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
775 IntTy = llvm::IntegerType::get(C, Width);
778 Value *Zero = llvm::Constant::getNullValue(IntTy);
779 return CGF.
Builder.CreateICmpSLT(
V, Zero);
783 const CallExpr *E, llvm::Constant *calleeValue) {
799 llvm::Value *
X, llvm::Value *Y,
800 llvm::Value *&Carry) {
802 assert(
X->getType() == Y->getType() &&
803 "Arguments must be the same type. (Did you forget to make sure both "
804 "arguments have the same integer width?)");
807 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
808 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
809 return CGF.
Builder.CreateExtractValue(Tmp, 0);
813 unsigned IntrinsicID,
816 llvm::MDNode *RNode = MDHelper.createRange(
APInt(32, low),
APInt(32, high));
818 llvm::Instruction *Call = CGF.
Builder.CreateCall(F);
819 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
820 Call->setMetadata(llvm::LLVMContext::MD_noundef,
826 struct WidthAndSignedness {
832 static WidthAndSignedness
846 static struct WidthAndSignedness
848 assert(Types.size() > 0 &&
"Empty list of types.");
852 for (
const auto &
Type : Types) {
861 for (
const auto &
Type : Types) {
863 if (Width < MinWidth) {
872 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
873 return Builder.CreateCall(CGM.getIntrinsic(inst, {ArgValue->getType()}),
883 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
888 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
892 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
893 llvm::IntegerType *ResType,
894 llvm::Value *EmittedE,
898 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE, IsDynamic);
899 return ConstantInt::get(ResType, ObjectSize,
true);
905 getLangOpts().getStrictFlexArraysLevel();
906 uint32_t FieldNo = 0;
912 if ((Name.empty() || FD->getNameAsString() == Name) &&
914 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
923 if (
const FieldDecl *Field = FindFlexibleArrayMemberField(
942 if (FD->getType()->isCountAttributedType())
954 CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *E,
unsigned Type,
955 llvm::IntegerType *ResType) {
984 const Expr *Idx =
nullptr;
986 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
987 UO && UO->getOpcode() == UO_AddrOf) {
989 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
990 Base = ASE->getBase()->IgnoreParenImpCasts();
993 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
994 int64_t Val = IL->getValue().getSExtValue();
1010 std::string FAMName;
1011 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
1013 const ValueDecl *VD = ME->getMemberDecl();
1016 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
1018 QualType Ty = DRE->getDecl()->getType();
1056 FindFlexibleArrayMemberField(Ctx, OuterRD, FAMName,
Offset);
1064 const FieldDecl *CountedByFD = FindCountedByField(FAMDecl);
1071 Value *CountedByInst = EmitCountedByFieldExpr(
Base, FAMDecl, CountedByFD);
1075 CountedByInst = Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1078 Value *IdxInst =
nullptr;
1085 IdxInst = EmitAnyExprToTemp(Idx).getScalarVal();
1086 IdxInst = Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1091 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1097 llvm::Constant *ElemSize =
1098 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1100 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1101 FAMSize = Builder.CreateIntCast(FAMSize, ResType, IsSigned);
1102 Value *Res = FAMSize;
1104 if (isa<DeclRefExpr>(
Base)) {
1109 llvm::Constant *FAMOffset = ConstantInt::get(ResType,
Offset, IsSigned);
1110 Value *OffsetAndFAMSize =
1111 Builder.CreateAdd(FAMOffset, Res,
"", !IsSigned, IsSigned);
1114 llvm::Constant *SizeofStruct =
1120 ? Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
1121 OffsetAndFAMSize, SizeofStruct)
1123 OffsetAndFAMSize, SizeofStruct);
1129 Value *Cmp = Builder.CreateIsNotNeg(CountedByInst);
1131 Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(IdxInst), Cmp);
1133 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1146 CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
1147 llvm::IntegerType *ResType,
1148 llvm::Value *EmittedE,
bool IsDynamic) {
1152 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
1153 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1154 if (Param !=
nullptr && PS !=
nullptr &&
1156 auto Iter = SizeArguments.find(Param);
1157 assert(
Iter != SizeArguments.end());
1160 auto DIter = LocalDeclMap.find(D);
1161 assert(DIter != LocalDeclMap.end());
1163 return EmitLoadOfScalar(DIter->second,
false,
1171 if (
Value *
V = emitFlexibleArrayMemberSize(E,
Type, ResType))
1181 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1182 assert(Ptr->
getType()->isPointerTy() &&
1183 "Non-pointer passed to __builtin_object_size?");
1186 CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->
getType()});
1191 Value *NullIsUnknown = Builder.getTrue();
1193 return Builder.CreateCall(F, {Ptr,
Min, NullIsUnknown,
Dynamic});
1199 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1200 enum InterlockingKind : uint8_t {
1209 InterlockingKind Interlocking;
1212 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1217 llvm::IntrinsicInst *getConvergenceToken(llvm::BasicBlock *BB) {
1218 for (
auto &I : *BB) {
1219 auto *II = dyn_cast<llvm::IntrinsicInst>(&I);
1220 if (II && isConvergenceControlIntrinsic(II->getIntrinsicID()))
1229 CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input,
1230 llvm::Value *ParentToken) {
1231 llvm::Value *bundleArgs[] = {ParentToken};
1232 llvm::OperandBundleDef OB(
"convergencectrl", bundleArgs);
1233 auto Output = llvm::CallBase::addOperandBundle(
1234 Input, llvm::LLVMContext::OB_convergencectrl, OB, Input);
1235 Input->replaceAllUsesWith(Output);
1236 Input->eraseFromParent();
1240 llvm::IntrinsicInst *
1241 CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB,
1242 llvm::Value *ParentToken) {
1243 CGBuilderTy::InsertPoint IP = Builder.saveIP();
1244 Builder.SetInsertPoint(&BB->front());
1245 auto CB = Builder.CreateIntrinsic(
1246 llvm::Intrinsic::experimental_convergence_loop, {}, {});
1247 Builder.restoreIP(IP);
1249 auto I = addConvergenceControlToken(CB, ParentToken);
1250 return cast<llvm::IntrinsicInst>(I);
1253 llvm::IntrinsicInst *
1254 CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
1255 auto *BB = &F->getEntryBlock();
1256 auto *token = getConvergenceToken(BB);
1264 CGBuilderTy::InsertPoint IP = Builder.saveIP();
1265 Builder.SetInsertPoint(&BB->front());
1266 auto I = Builder.CreateIntrinsic(
1267 llvm::Intrinsic::experimental_convergence_entry, {}, {});
1268 assert(isa<llvm::IntrinsicInst>(I));
1269 Builder.restoreIP(IP);
1271 return cast<llvm::IntrinsicInst>(I);
1274 llvm::IntrinsicInst *
1275 CodeGenFunction::getOrEmitConvergenceLoopToken(
const LoopInfo *LI) {
1276 assert(LI !=
nullptr);
1278 auto *token = getConvergenceToken(LI->
getHeader());
1282 llvm::IntrinsicInst *PII =
1284 ? emitConvergenceLoopToken(
1286 : getOrEmitConvergenceEntryToken(LI->getHeader()->getParent());
1288 return emitConvergenceLoopToken(LI->
getHeader(), PII);
1293 llvm::Value *ParentToken =
1295 ? getOrEmitConvergenceLoopToken(&LoopStack.getInfo())
1296 : getOrEmitConvergenceEntryToken(Input->getFunction());
1297 return addConvergenceControlToken(Input, ParentToken);
1300 BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1301 switch (BuiltinID) {
1303 case Builtin::BI_bittest:
1304 return {TestOnly, Unlocked,
false};
1305 case Builtin::BI_bittestandcomplement:
1306 return {Complement, Unlocked,
false};
1307 case Builtin::BI_bittestandreset:
1308 return {Reset, Unlocked,
false};
1309 case Builtin::BI_bittestandset:
1310 return {
Set, Unlocked,
false};
1311 case Builtin::BI_interlockedbittestandreset:
1312 return {Reset, Sequential,
false};
1313 case Builtin::BI_interlockedbittestandset:
1314 return {
Set, Sequential,
false};
1317 case Builtin::BI_bittest64:
1318 return {TestOnly, Unlocked,
true};
1319 case Builtin::BI_bittestandcomplement64:
1320 return {Complement, Unlocked,
true};
1321 case Builtin::BI_bittestandreset64:
1322 return {Reset, Unlocked,
true};
1323 case Builtin::BI_bittestandset64:
1324 return {
Set, Unlocked,
true};
1325 case Builtin::BI_interlockedbittestandreset64:
1326 return {Reset, Sequential,
true};
1327 case Builtin::BI_interlockedbittestandset64:
1328 return {
Set, Sequential,
true};
1331 case Builtin::BI_interlockedbittestandset_acq:
1332 return {
Set, Acquire,
false};
1333 case Builtin::BI_interlockedbittestandset_rel:
1334 return {
Set, Release,
false};
1335 case Builtin::BI_interlockedbittestandset_nf:
1336 return {
Set, NoFence,
false};
1337 case Builtin::BI_interlockedbittestandreset_acq:
1338 return {Reset, Acquire,
false};
1339 case Builtin::BI_interlockedbittestandreset_rel:
1340 return {Reset, Release,
false};
1341 case Builtin::BI_interlockedbittestandreset_nf:
1342 return {Reset, NoFence,
false};
1344 llvm_unreachable(
"expected only bittest intrinsics");
1349 case BitTest::TestOnly:
return '\0';
1350 case BitTest::Complement:
return 'c';
1351 case BitTest::Reset:
return 'r';
1352 case BitTest::Set:
return 's';
1354 llvm_unreachable(
"invalid action");
1362 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1366 raw_svector_ostream AsmOS(
Asm);
1367 if (BT.Interlocking != BitTest::Unlocked)
1372 AsmOS << SizeSuffix <<
" $2, ($1)";
1375 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1377 if (!MachineClobbers.empty()) {
1379 Constraints += MachineClobbers;
1381 llvm::IntegerType *IntType = llvm::IntegerType::get(
1384 llvm::FunctionType *FTy =
1385 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1387 llvm::InlineAsm *IA =
1388 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1389 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1392 static llvm::AtomicOrdering
1395 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1396 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1397 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1398 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1399 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1401 llvm_unreachable(
"invalid interlocking");
1414 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1426 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1429 ByteIndex,
"bittest.byteaddr"),
1433 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1436 Value *Mask =
nullptr;
1437 if (BT.Action != BitTest::TestOnly) {
1438 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1445 Value *OldByte =
nullptr;
1446 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1449 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1450 if (BT.Action == BitTest::Reset) {
1451 Mask = CGF.
Builder.CreateNot(Mask);
1458 Value *NewByte =
nullptr;
1459 switch (BT.Action) {
1460 case BitTest::TestOnly:
1463 case BitTest::Complement:
1464 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1466 case BitTest::Reset:
1467 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1470 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1479 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1481 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1490 raw_svector_ostream AsmOS(
Asm);
1491 llvm::IntegerType *RetType = CGF.
Int32Ty;
1493 switch (BuiltinID) {
1494 case clang::PPC::BI__builtin_ppc_ldarx:
1498 case clang::PPC::BI__builtin_ppc_lwarx:
1502 case clang::PPC::BI__builtin_ppc_lharx:
1506 case clang::PPC::BI__builtin_ppc_lbarx:
1511 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1514 AsmOS <<
"$0, ${1:y}";
1516 std::string Constraints =
"=r,*Z,~{memory}";
1518 if (!MachineClobbers.empty()) {
1520 Constraints += MachineClobbers;
1524 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1526 llvm::InlineAsm *IA =
1527 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1528 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1530 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1535 enum class MSVCSetJmpKind {
1547 llvm::Value *Arg1 =
nullptr;
1548 llvm::Type *Arg1Ty =
nullptr;
1550 bool IsVarArg =
false;
1551 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1554 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1557 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1560 Arg1 = CGF.
Builder.CreateCall(
1563 Arg1 = CGF.
Builder.CreateCall(
1565 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1569 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1570 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1572 llvm::Attribute::ReturnsTwice);
1574 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1575 ReturnsTwiceAttr,
true);
1577 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1579 llvm::Value *Args[] = {Buf, Arg1};
1581 CB->setAttributes(ReturnsTwiceAttr);
1591 _InterlockedDecrement,
1592 _InterlockedExchange,
1593 _InterlockedExchangeAdd,
1594 _InterlockedExchangeSub,
1595 _InterlockedIncrement,
1598 _InterlockedExchangeAdd_acq,
1599 _InterlockedExchangeAdd_rel,
1600 _InterlockedExchangeAdd_nf,
1601 _InterlockedExchange_acq,
1602 _InterlockedExchange_rel,
1603 _InterlockedExchange_nf,
1604 _InterlockedCompareExchange_acq,
1605 _InterlockedCompareExchange_rel,
1606 _InterlockedCompareExchange_nf,
1607 _InterlockedCompareExchange128,
1608 _InterlockedCompareExchange128_acq,
1609 _InterlockedCompareExchange128_rel,
1610 _InterlockedCompareExchange128_nf,
1614 _InterlockedXor_acq,
1615 _InterlockedXor_rel,
1617 _InterlockedAnd_acq,
1618 _InterlockedAnd_rel,
1620 _InterlockedIncrement_acq,
1621 _InterlockedIncrement_rel,
1622 _InterlockedIncrement_nf,
1623 _InterlockedDecrement_acq,
1624 _InterlockedDecrement_rel,
1625 _InterlockedDecrement_nf,
1629 static std::optional<CodeGenFunction::MSVCIntrin>
1632 switch (BuiltinID) {
1634 return std::nullopt;
1635 case clang::ARM::BI_BitScanForward:
1636 case clang::ARM::BI_BitScanForward64:
1637 return MSVCIntrin::_BitScanForward;
1638 case clang::ARM::BI_BitScanReverse:
1639 case clang::ARM::BI_BitScanReverse64:
1640 return MSVCIntrin::_BitScanReverse;
1641 case clang::ARM::BI_InterlockedAnd64:
1642 return MSVCIntrin::_InterlockedAnd;
1643 case clang::ARM::BI_InterlockedExchange64:
1644 return MSVCIntrin::_InterlockedExchange;
1645 case clang::ARM::BI_InterlockedExchangeAdd64:
1646 return MSVCIntrin::_InterlockedExchangeAdd;
1647 case clang::ARM::BI_InterlockedExchangeSub64:
1648 return MSVCIntrin::_InterlockedExchangeSub;
1649 case clang::ARM::BI_InterlockedOr64:
1650 return MSVCIntrin::_InterlockedOr;
1651 case clang::ARM::BI_InterlockedXor64:
1652 return MSVCIntrin::_InterlockedXor;
1653 case clang::ARM::BI_InterlockedDecrement64:
1654 return MSVCIntrin::_InterlockedDecrement;
1655 case clang::ARM::BI_InterlockedIncrement64:
1656 return MSVCIntrin::_InterlockedIncrement;
1657 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1658 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1659 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1660 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1661 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1662 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1663 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1664 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1665 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1666 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1667 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1668 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1669 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1670 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1671 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1672 case clang::ARM::BI_InterlockedExchange8_acq:
1673 case clang::ARM::BI_InterlockedExchange16_acq:
1674 case clang::ARM::BI_InterlockedExchange_acq:
1675 case clang::ARM::BI_InterlockedExchange64_acq:
1676 return MSVCIntrin::_InterlockedExchange_acq;
1677 case clang::ARM::BI_InterlockedExchange8_rel:
1678 case clang::ARM::BI_InterlockedExchange16_rel:
1679 case clang::ARM::BI_InterlockedExchange_rel:
1680 case clang::ARM::BI_InterlockedExchange64_rel:
1681 return MSVCIntrin::_InterlockedExchange_rel;
1682 case clang::ARM::BI_InterlockedExchange8_nf:
1683 case clang::ARM::BI_InterlockedExchange16_nf:
1684 case clang::ARM::BI_InterlockedExchange_nf:
1685 case clang::ARM::BI_InterlockedExchange64_nf:
1686 return MSVCIntrin::_InterlockedExchange_nf;
1687 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1688 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1689 case clang::ARM::BI_InterlockedCompareExchange_acq:
1690 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1691 return MSVCIntrin::_InterlockedCompareExchange_acq;
1692 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1693 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1694 case clang::ARM::BI_InterlockedCompareExchange_rel:
1695 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1696 return MSVCIntrin::_InterlockedCompareExchange_rel;
1697 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1698 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1699 case clang::ARM::BI_InterlockedCompareExchange_nf:
1700 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1701 return MSVCIntrin::_InterlockedCompareExchange_nf;
1702 case clang::ARM::BI_InterlockedOr8_acq:
1703 case clang::ARM::BI_InterlockedOr16_acq:
1704 case clang::ARM::BI_InterlockedOr_acq:
1705 case clang::ARM::BI_InterlockedOr64_acq:
1706 return MSVCIntrin::_InterlockedOr_acq;
1707 case clang::ARM::BI_InterlockedOr8_rel:
1708 case clang::ARM::BI_InterlockedOr16_rel:
1709 case clang::ARM::BI_InterlockedOr_rel:
1710 case clang::ARM::BI_InterlockedOr64_rel:
1711 return MSVCIntrin::_InterlockedOr_rel;
1712 case clang::ARM::BI_InterlockedOr8_nf:
1713 case clang::ARM::BI_InterlockedOr16_nf:
1714 case clang::ARM::BI_InterlockedOr_nf:
1715 case clang::ARM::BI_InterlockedOr64_nf:
1716 return MSVCIntrin::_InterlockedOr_nf;
1717 case clang::ARM::BI_InterlockedXor8_acq:
1718 case clang::ARM::BI_InterlockedXor16_acq:
1719 case clang::ARM::BI_InterlockedXor_acq:
1720 case clang::ARM::BI_InterlockedXor64_acq:
1721 return MSVCIntrin::_InterlockedXor_acq;
1722 case clang::ARM::BI_InterlockedXor8_rel:
1723 case clang::ARM::BI_InterlockedXor16_rel:
1724 case clang::ARM::BI_InterlockedXor_rel:
1725 case clang::ARM::BI_InterlockedXor64_rel:
1726 return MSVCIntrin::_InterlockedXor_rel;
1727 case clang::ARM::BI_InterlockedXor8_nf:
1728 case clang::ARM::BI_InterlockedXor16_nf:
1729 case clang::ARM::BI_InterlockedXor_nf:
1730 case clang::ARM::BI_InterlockedXor64_nf:
1731 return MSVCIntrin::_InterlockedXor_nf;
1732 case clang::ARM::BI_InterlockedAnd8_acq:
1733 case clang::ARM::BI_InterlockedAnd16_acq:
1734 case clang::ARM::BI_InterlockedAnd_acq:
1735 case clang::ARM::BI_InterlockedAnd64_acq:
1736 return MSVCIntrin::_InterlockedAnd_acq;
1737 case clang::ARM::BI_InterlockedAnd8_rel:
1738 case clang::ARM::BI_InterlockedAnd16_rel:
1739 case clang::ARM::BI_InterlockedAnd_rel:
1740 case clang::ARM::BI_InterlockedAnd64_rel:
1741 return MSVCIntrin::_InterlockedAnd_rel;
1742 case clang::ARM::BI_InterlockedAnd8_nf:
1743 case clang::ARM::BI_InterlockedAnd16_nf:
1744 case clang::ARM::BI_InterlockedAnd_nf:
1745 case clang::ARM::BI_InterlockedAnd64_nf:
1746 return MSVCIntrin::_InterlockedAnd_nf;
1747 case clang::ARM::BI_InterlockedIncrement16_acq:
1748 case clang::ARM::BI_InterlockedIncrement_acq:
1749 case clang::ARM::BI_InterlockedIncrement64_acq:
1750 return MSVCIntrin::_InterlockedIncrement_acq;
1751 case clang::ARM::BI_InterlockedIncrement16_rel:
1752 case clang::ARM::BI_InterlockedIncrement_rel:
1753 case clang::ARM::BI_InterlockedIncrement64_rel:
1754 return MSVCIntrin::_InterlockedIncrement_rel;
1755 case clang::ARM::BI_InterlockedIncrement16_nf:
1756 case clang::ARM::BI_InterlockedIncrement_nf:
1757 case clang::ARM::BI_InterlockedIncrement64_nf:
1758 return MSVCIntrin::_InterlockedIncrement_nf;
1759 case clang::ARM::BI_InterlockedDecrement16_acq:
1760 case clang::ARM::BI_InterlockedDecrement_acq:
1761 case clang::ARM::BI_InterlockedDecrement64_acq:
1762 return MSVCIntrin::_InterlockedDecrement_acq;
1763 case clang::ARM::BI_InterlockedDecrement16_rel:
1764 case clang::ARM::BI_InterlockedDecrement_rel:
1765 case clang::ARM::BI_InterlockedDecrement64_rel:
1766 return MSVCIntrin::_InterlockedDecrement_rel;
1767 case clang::ARM::BI_InterlockedDecrement16_nf:
1768 case clang::ARM::BI_InterlockedDecrement_nf:
1769 case clang::ARM::BI_InterlockedDecrement64_nf:
1770 return MSVCIntrin::_InterlockedDecrement_nf;
1772 llvm_unreachable(
"must return from switch");
1775 static std::optional<CodeGenFunction::MSVCIntrin>
1778 switch (BuiltinID) {
1780 return std::nullopt;
1781 case clang::AArch64::BI_BitScanForward:
1782 case clang::AArch64::BI_BitScanForward64:
1783 return MSVCIntrin::_BitScanForward;
1784 case clang::AArch64::BI_BitScanReverse:
1785 case clang::AArch64::BI_BitScanReverse64:
1786 return MSVCIntrin::_BitScanReverse;
1787 case clang::AArch64::BI_InterlockedAnd64:
1788 return MSVCIntrin::_InterlockedAnd;
1789 case clang::AArch64::BI_InterlockedExchange64:
1790 return MSVCIntrin::_InterlockedExchange;
1791 case clang::AArch64::BI_InterlockedExchangeAdd64:
1792 return MSVCIntrin::_InterlockedExchangeAdd;
1793 case clang::AArch64::BI_InterlockedExchangeSub64:
1794 return MSVCIntrin::_InterlockedExchangeSub;
1795 case clang::AArch64::BI_InterlockedOr64:
1796 return MSVCIntrin::_InterlockedOr;
1797 case clang::AArch64::BI_InterlockedXor64:
1798 return MSVCIntrin::_InterlockedXor;
1799 case clang::AArch64::BI_InterlockedDecrement64:
1800 return MSVCIntrin::_InterlockedDecrement;
1801 case clang::AArch64::BI_InterlockedIncrement64:
1802 return MSVCIntrin::_InterlockedIncrement;
1803 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1804 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1805 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1806 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1807 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1808 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1809 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1810 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1811 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1812 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1813 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1814 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1815 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1816 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1817 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1818 case clang::AArch64::BI_InterlockedExchange8_acq:
1819 case clang::AArch64::BI_InterlockedExchange16_acq:
1820 case clang::AArch64::BI_InterlockedExchange_acq:
1821 case clang::AArch64::BI_InterlockedExchange64_acq:
1822 return MSVCIntrin::_InterlockedExchange_acq;
1823 case clang::AArch64::BI_InterlockedExchange8_rel:
1824 case clang::AArch64::BI_InterlockedExchange16_rel:
1825 case clang::AArch64::BI_InterlockedExchange_rel:
1826 case clang::AArch64::BI_InterlockedExchange64_rel:
1827 return MSVCIntrin::_InterlockedExchange_rel;
1828 case clang::AArch64::BI_InterlockedExchange8_nf:
1829 case clang::AArch64::BI_InterlockedExchange16_nf:
1830 case clang::AArch64::BI_InterlockedExchange_nf:
1831 case clang::AArch64::BI_InterlockedExchange64_nf:
1832 return MSVCIntrin::_InterlockedExchange_nf;
1833 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1834 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1835 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1836 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1837 return MSVCIntrin::_InterlockedCompareExchange_acq;
1838 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1839 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1840 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1841 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1842 return MSVCIntrin::_InterlockedCompareExchange_rel;
1843 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1844 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1845 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1846 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1847 return MSVCIntrin::_InterlockedCompareExchange_nf;
1848 case clang::AArch64::BI_InterlockedCompareExchange128:
1849 return MSVCIntrin::_InterlockedCompareExchange128;
1850 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1851 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1852 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1853 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1854 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1855 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1856 case clang::AArch64::BI_InterlockedOr8_acq:
1857 case clang::AArch64::BI_InterlockedOr16_acq:
1858 case clang::AArch64::BI_InterlockedOr_acq:
1859 case clang::AArch64::BI_InterlockedOr64_acq:
1860 return MSVCIntrin::_InterlockedOr_acq;
1861 case clang::AArch64::BI_InterlockedOr8_rel:
1862 case clang::AArch64::BI_InterlockedOr16_rel:
1863 case clang::AArch64::BI_InterlockedOr_rel:
1864 case clang::AArch64::BI_InterlockedOr64_rel:
1865 return MSVCIntrin::_InterlockedOr_rel;
1866 case clang::AArch64::BI_InterlockedOr8_nf:
1867 case clang::AArch64::BI_InterlockedOr16_nf:
1868 case clang::AArch64::BI_InterlockedOr_nf:
1869 case clang::AArch64::BI_InterlockedOr64_nf:
1870 return MSVCIntrin::_InterlockedOr_nf;
1871 case clang::AArch64::BI_InterlockedXor8_acq:
1872 case clang::AArch64::BI_InterlockedXor16_acq:
1873 case clang::AArch64::BI_InterlockedXor_acq:
1874 case clang::AArch64::BI_InterlockedXor64_acq:
1875 return MSVCIntrin::_InterlockedXor_acq;
1876 case clang::AArch64::BI_InterlockedXor8_rel:
1877 case clang::AArch64::BI_InterlockedXor16_rel:
1878 case clang::AArch64::BI_InterlockedXor_rel:
1879 case clang::AArch64::BI_InterlockedXor64_rel:
1880 return MSVCIntrin::_InterlockedXor_rel;
1881 case clang::AArch64::BI_InterlockedXor8_nf:
1882 case clang::AArch64::BI_InterlockedXor16_nf:
1883 case clang::AArch64::BI_InterlockedXor_nf:
1884 case clang::AArch64::BI_InterlockedXor64_nf:
1885 return MSVCIntrin::_InterlockedXor_nf;
1886 case clang::AArch64::BI_InterlockedAnd8_acq:
1887 case clang::AArch64::BI_InterlockedAnd16_acq:
1888 case clang::AArch64::BI_InterlockedAnd_acq:
1889 case clang::AArch64::BI_InterlockedAnd64_acq:
1890 return MSVCIntrin::_InterlockedAnd_acq;
1891 case clang::AArch64::BI_InterlockedAnd8_rel:
1892 case clang::AArch64::BI_InterlockedAnd16_rel:
1893 case clang::AArch64::BI_InterlockedAnd_rel:
1894 case clang::AArch64::BI_InterlockedAnd64_rel:
1895 return MSVCIntrin::_InterlockedAnd_rel;
1896 case clang::AArch64::BI_InterlockedAnd8_nf:
1897 case clang::AArch64::BI_InterlockedAnd16_nf:
1898 case clang::AArch64::BI_InterlockedAnd_nf:
1899 case clang::AArch64::BI_InterlockedAnd64_nf:
1900 return MSVCIntrin::_InterlockedAnd_nf;
1901 case clang::AArch64::BI_InterlockedIncrement16_acq:
1902 case clang::AArch64::BI_InterlockedIncrement_acq:
1903 case clang::AArch64::BI_InterlockedIncrement64_acq:
1904 return MSVCIntrin::_InterlockedIncrement_acq;
1905 case clang::AArch64::BI_InterlockedIncrement16_rel:
1906 case clang::AArch64::BI_InterlockedIncrement_rel:
1907 case clang::AArch64::BI_InterlockedIncrement64_rel:
1908 return MSVCIntrin::_InterlockedIncrement_rel;
1909 case clang::AArch64::BI_InterlockedIncrement16_nf:
1910 case clang::AArch64::BI_InterlockedIncrement_nf:
1911 case clang::AArch64::BI_InterlockedIncrement64_nf:
1912 return MSVCIntrin::_InterlockedIncrement_nf;
1913 case clang::AArch64::BI_InterlockedDecrement16_acq:
1914 case clang::AArch64::BI_InterlockedDecrement_acq:
1915 case clang::AArch64::BI_InterlockedDecrement64_acq:
1916 return MSVCIntrin::_InterlockedDecrement_acq;
1917 case clang::AArch64::BI_InterlockedDecrement16_rel:
1918 case clang::AArch64::BI_InterlockedDecrement_rel:
1919 case clang::AArch64::BI_InterlockedDecrement64_rel:
1920 return MSVCIntrin::_InterlockedDecrement_rel;
1921 case clang::AArch64::BI_InterlockedDecrement16_nf:
1922 case clang::AArch64::BI_InterlockedDecrement_nf:
1923 case clang::AArch64::BI_InterlockedDecrement64_nf:
1924 return MSVCIntrin::_InterlockedDecrement_nf;
1926 llvm_unreachable(
"must return from switch");
1929 static std::optional<CodeGenFunction::MSVCIntrin>
1932 switch (BuiltinID) {
1934 return std::nullopt;
1935 case clang::X86::BI_BitScanForward:
1936 case clang::X86::BI_BitScanForward64:
1937 return MSVCIntrin::_BitScanForward;
1938 case clang::X86::BI_BitScanReverse:
1939 case clang::X86::BI_BitScanReverse64:
1940 return MSVCIntrin::_BitScanReverse;
1941 case clang::X86::BI_InterlockedAnd64:
1942 return MSVCIntrin::_InterlockedAnd;
1943 case clang::X86::BI_InterlockedCompareExchange128:
1944 return MSVCIntrin::_InterlockedCompareExchange128;
1945 case clang::X86::BI_InterlockedExchange64:
1946 return MSVCIntrin::_InterlockedExchange;
1947 case clang::X86::BI_InterlockedExchangeAdd64:
1948 return MSVCIntrin::_InterlockedExchangeAdd;
1949 case clang::X86::BI_InterlockedExchangeSub64:
1950 return MSVCIntrin::_InterlockedExchangeSub;
1951 case clang::X86::BI_InterlockedOr64:
1952 return MSVCIntrin::_InterlockedOr;
1953 case clang::X86::BI_InterlockedXor64:
1954 return MSVCIntrin::_InterlockedXor;
1955 case clang::X86::BI_InterlockedDecrement64:
1956 return MSVCIntrin::_InterlockedDecrement;
1957 case clang::X86::BI_InterlockedIncrement64:
1958 return MSVCIntrin::_InterlockedIncrement;
1960 llvm_unreachable(
"must return from switch");
1966 switch (BuiltinID) {
1967 case MSVCIntrin::_BitScanForward:
1968 case MSVCIntrin::_BitScanReverse: {
1972 llvm::Type *ArgType = ArgValue->
getType();
1974 llvm::Type *ResultType = ConvertType(E->
getType());
1976 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1977 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1978 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1980 BasicBlock *
Begin = Builder.GetInsertBlock();
1981 BasicBlock *
End = createBasicBlock(
"bitscan_end", this->CurFn);
1982 Builder.SetInsertPoint(
End);
1983 PHINode *Result = Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1985 Builder.SetInsertPoint(
Begin);
1986 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
1987 BasicBlock *NotZero = createBasicBlock(
"bitscan_not_zero", this->CurFn);
1988 Builder.CreateCondBr(IsZero,
End, NotZero);
1989 Result->addIncoming(ResZero,
Begin);
1991 Builder.SetInsertPoint(NotZero);
1993 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1994 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1995 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1996 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
1997 Builder.CreateStore(ZeroCount, IndexAddress,
false);
1999 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
2000 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
2002 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2003 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
2004 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
2005 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
2006 Builder.CreateStore(Index, IndexAddress,
false);
2008 Builder.CreateBr(
End);
2009 Result->addIncoming(ResOne, NotZero);
2011 Builder.SetInsertPoint(
End);
2014 case MSVCIntrin::_InterlockedAnd:
2016 case MSVCIntrin::_InterlockedExchange:
2018 case MSVCIntrin::_InterlockedExchangeAdd:
2020 case MSVCIntrin::_InterlockedExchangeSub:
2022 case MSVCIntrin::_InterlockedOr:
2024 case MSVCIntrin::_InterlockedXor:
2026 case MSVCIntrin::_InterlockedExchangeAdd_acq:
2028 AtomicOrdering::Acquire);
2029 case MSVCIntrin::_InterlockedExchangeAdd_rel:
2031 AtomicOrdering::Release);
2032 case MSVCIntrin::_InterlockedExchangeAdd_nf:
2034 AtomicOrdering::Monotonic);
2035 case MSVCIntrin::_InterlockedExchange_acq:
2037 AtomicOrdering::Acquire);
2038 case MSVCIntrin::_InterlockedExchange_rel:
2040 AtomicOrdering::Release);
2041 case MSVCIntrin::_InterlockedExchange_nf:
2043 AtomicOrdering::Monotonic);
2044 case MSVCIntrin::_InterlockedCompareExchange_acq:
2046 case MSVCIntrin::_InterlockedCompareExchange_rel:
2048 case MSVCIntrin::_InterlockedCompareExchange_nf:
2050 case MSVCIntrin::_InterlockedCompareExchange128:
2052 *
this, E, AtomicOrdering::SequentiallyConsistent);
2053 case MSVCIntrin::_InterlockedCompareExchange128_acq:
2055 case MSVCIntrin::_InterlockedCompareExchange128_rel:
2057 case MSVCIntrin::_InterlockedCompareExchange128_nf:
2059 case MSVCIntrin::_InterlockedOr_acq:
2061 AtomicOrdering::Acquire);
2062 case MSVCIntrin::_InterlockedOr_rel:
2064 AtomicOrdering::Release);
2065 case MSVCIntrin::_InterlockedOr_nf:
2067 AtomicOrdering::Monotonic);
2068 case MSVCIntrin::_InterlockedXor_acq:
2070 AtomicOrdering::Acquire);
2071 case MSVCIntrin::_InterlockedXor_rel:
2073 AtomicOrdering::Release);
2074 case MSVCIntrin::_InterlockedXor_nf:
2076 AtomicOrdering::Monotonic);
2077 case MSVCIntrin::_InterlockedAnd_acq:
2079 AtomicOrdering::Acquire);
2080 case MSVCIntrin::_InterlockedAnd_rel:
2082 AtomicOrdering::Release);
2083 case MSVCIntrin::_InterlockedAnd_nf:
2085 AtomicOrdering::Monotonic);
2086 case MSVCIntrin::_InterlockedIncrement_acq:
2088 case MSVCIntrin::_InterlockedIncrement_rel:
2090 case MSVCIntrin::_InterlockedIncrement_nf:
2092 case MSVCIntrin::_InterlockedDecrement_acq:
2094 case MSVCIntrin::_InterlockedDecrement_rel:
2096 case MSVCIntrin::_InterlockedDecrement_nf:
2099 case MSVCIntrin::_InterlockedDecrement:
2101 case MSVCIntrin::_InterlockedIncrement:
2104 case MSVCIntrin::__fastfail: {
2108 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
2109 StringRef
Asm, Constraints;
2112 ErrorUnsupported(E,
"__fastfail call for this architecture");
2114 case llvm::Triple::x86:
2115 case llvm::Triple::x86_64:
2117 Constraints =
"{cx}";
2119 case llvm::Triple::thumb:
2121 Constraints =
"{r0}";
2123 case llvm::Triple::aarch64:
2124 Asm =
"brk #0xF003";
2125 Constraints =
"{w0}";
2127 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty},
false);
2128 llvm::InlineAsm *IA =
2129 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
2130 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2131 getLLVMContext(), llvm::AttributeList::FunctionIndex,
2132 llvm::Attribute::NoReturn);
2133 llvm::CallInst *CI = Builder.CreateCall(IA, EmitScalarExpr(E->
getArg(0)));
2134 CI->setAttributes(NoReturnAttr);
2138 llvm_unreachable(
"Incorrect MSVC intrinsic!");
2144 CallObjCArcUse(llvm::Value *
object) :
object(
object) {}
2155 assert((
Kind == BCK_CLZPassedZero ||
Kind == BCK_CTZPassedZero)
2156 &&
"Unsupported builtin check kind");
2158 Value *ArgValue = EmitScalarExpr(E);
2159 if (!SanOpts.has(SanitizerKind::Builtin))
2163 Value *Cond = Builder.CreateICmpNE(
2164 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2165 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2166 SanitizerHandler::InvalidBuiltin,
2168 llvm::ConstantInt::get(Builder.getInt8Ty(),
Kind)},
2174 return CGF.
Builder.CreateBinaryIntrinsic(
2176 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2180 bool SanitizeOverflow) {
2184 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2185 if (!VCI->isMinSignedValue())
2186 return EmitAbs(CGF, ArgValue,
true);
2191 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2192 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2193 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2194 Value *Result = CGF.
Builder.CreateExtractValue(ResultAndOverflow, 0);
2196 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2199 if (SanitizeOverflow) {
2200 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2201 SanitizerHandler::NegateOverflow,
2206 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2208 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2209 return CGF.
Builder.CreateSelect(CmpResult, Result, ArgValue,
"abs");
2214 QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8,
false);
2215 return C.getCanonicalType(UnsignedTy);
2225 raw_svector_ostream OS(Name);
2226 OS <<
"__os_log_helper";
2230 for (
const auto &Item : Layout.
Items)
2231 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2232 <<
int(Item.getDescriptorByte());
2235 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2245 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
2246 char Size = Layout.
Items[I].getSizeByte();
2253 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2255 ArgTys.emplace_back(ArgTy);
2264 CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, Args);
2265 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
2266 llvm::Function *Fn = llvm::Function::Create(
2267 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
2269 CGM.SetLLVMFunctionAttributes(
GlobalDecl(), FI, Fn,
false);
2270 CGM.SetLLVMFunctionAttributesForDefinition(
nullptr, Fn);
2271 Fn->setDoesNotThrow();
2274 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2275 Fn->addFnAttr(llvm::Attribute::NoInline);
2278 StartFunction(
GlobalDecl(), ReturnTy, Fn, FI, Args);
2284 Address BufAddr = makeNaturalAddressForPointer(
2285 Builder.CreateLoad(GetAddrOfLocalVar(Args[0]),
"buf"), Ctx.
VoidTy,
2288 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"summary"));
2290 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"numArgs"));
2293 for (
const auto &Item : Layout.
Items) {
2294 Builder.CreateStore(
2295 Builder.getInt8(Item.getDescriptorByte()),
2296 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"argDescriptor"));
2297 Builder.CreateStore(
2298 Builder.getInt8(Item.getSizeByte()),
2299 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"argSize"));
2302 if (!Size.getQuantity())
2305 Address Arg = GetAddrOfLocalVar(Args[I]);
2306 Address Addr = Builder.CreateConstByteGEP(BufAddr,
Offset,
"argData");
2308 Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
2320 "__builtin_os_log_format takes at least 2 arguments");
2331 for (
const auto &Item : Layout.
Items) {
2332 int Size = Item.getSizeByte();
2336 llvm::Value *ArgVal;
2340 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2341 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2342 ArgVal = llvm::Constant::getIntegerValue(Int64Ty,
llvm::APInt(64, Val));
2343 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2344 ArgVal = EmitScalarExpr(TheExpr,
false);
2349 auto LifetimeExtendObject = [&](
const Expr *E) {
2357 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
2362 if (TheExpr->getType()->isObjCRetainableType() &&
2363 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2364 assert(getEvaluationKind(TheExpr->getType()) ==
TEK_Scalar &&
2365 "Only scalar can be a ObjC retainable type");
2366 if (!isa<Constant>(ArgVal)) {
2370 RawAddress Addr = CreateMemTemp(Ty,
"os.log.arg", &Alloca);
2371 ArgVal = EmitARCRetain(Ty, ArgVal);
2372 Builder.CreateStore(ArgVal, Addr);
2373 pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty,
2379 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2380 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2384 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
2387 unsigned ArgValSize =
2388 CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
2389 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
2391 ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
2394 ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
2399 CGM.getTypes().arrangeBuiltinFunctionCall(Ctx.
VoidTy, Args);
2407 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2408 WidthAndSignedness ResultInfo) {
2409 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2410 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2411 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2416 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2418 WidthAndSignedness ResultInfo) {
2420 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2421 "Cannot specialize this multiply");
2426 llvm::Value *HasOverflow;
2428 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2433 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2434 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Result->getType(), IntMax);
2436 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(Result, IntMaxValue);
2437 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2449 WidthAndSignedness Op1Info,
2450 WidthAndSignedness Op2Info,
2451 WidthAndSignedness ResultInfo) {
2452 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2453 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2454 Op1Info.Signed != Op2Info.Signed;
2461 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2462 WidthAndSignedness Op2Info,
2464 WidthAndSignedness ResultInfo) {
2466 Op2Info, ResultInfo) &&
2467 "Not a mixed-sign multipliction we can specialize");
2470 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2471 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2474 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2475 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2478 if (SignedOpWidth < UnsignedOpWidth)
2480 if (UnsignedOpWidth < SignedOpWidth)
2483 llvm::Type *OpTy =
Signed->getType();
2484 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2487 unsigned OpWidth =
std::max(Op1Info.Width, Op2Info.Width);
2490 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2491 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2492 llvm::Value *AbsSigned =
2493 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2496 llvm::Value *UnsignedOverflow;
2497 llvm::Value *UnsignedResult =
2501 llvm::Value *Overflow, *Result;
2502 if (ResultInfo.Signed) {
2506 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2507 llvm::Value *MaxResult =
2508 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2509 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2510 llvm::Value *SignedOverflow =
2511 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2512 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2515 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2516 llvm::Value *SignedResult =
2517 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2518 Result = CGF.
Builder.CreateTrunc(SignedResult, ResTy);
2521 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2522 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2523 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2524 if (ResultInfo.Width < OpWidth) {
2526 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2527 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2528 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2529 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2533 Result = CGF.
Builder.CreateSelect(
2534 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2536 Result = CGF.
Builder.CreateTrunc(Result, ResTy);
2538 assert(Overflow && Result &&
"Missing overflow or result");
2549 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2558 if (!Seen.insert(
Record).second)
2561 assert(
Record->hasDefinition() &&
2562 "Incomplete types should already be diagnosed");
2564 if (
Record->isDynamicClass())
2584 llvm::Value *Src = EmitScalarExpr(E->
getArg(0));
2585 llvm::Value *ShiftAmt = EmitScalarExpr(E->
getArg(1));
2589 llvm::Type *Ty = Src->getType();
2590 ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty,
false);
2593 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2594 Function *F = CGM.getIntrinsic(IID, Ty);
2595 return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
2600 switch (BuiltinID) {
2601 #define MUTATE_LDBL(func) \
2602 case Builtin::BI__builtin_##func##l: \
2603 return Builtin::BI__builtin_##func##f128;
2672 if (CGF.
Builder.getIsFPConstrained() &&
2673 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2685 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2688 for (
auto &&FormalTy : FnTy->params())
2689 Args.push_back(llvm::PoisonValue::get(FormalTy));
2702 !Result.hasSideEffects()) {
2703 if (Result.Val.isInt())
2704 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
2705 Result.Val.getInt()));
2706 if (Result.Val.isFloat())
2707 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
2708 Result.Val.getFloat()));
2717 if (getTarget().getTriple().isPPC64() &&
2718 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2725 const unsigned BuiltinIDIfNoAsmLabel =
2726 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2728 std::optional<bool> ErrnoOverriden;
2734 if (OP.hasMathErrnoOverride())
2735 ErrnoOverriden = OP.getMathErrnoOverride();
2739 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2744 bool ErrnoOverridenToFalseWithOpt =
2745 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2746 CGM.getCodeGenOpts().OptimizationLevel != 0;
2756 getContext().BuiltinInfo.isConst(BuiltinID);
2763 switch (BuiltinID) {
2764 case Builtin::BI__builtin_fma:
2765 case Builtin::BI__builtin_fmaf:
2766 case Builtin::BI__builtin_fmal:
2767 case Builtin::BIfma:
2768 case Builtin::BIfmaf:
2769 case Builtin::BIfmal: {
2770 auto &Trip = CGM.getTriple();
2771 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2779 bool ConstWithoutErrnoAndExceptions =
2780 getContext().BuiltinInfo.isConstWithoutErrnoAndExceptions(BuiltinID);
2781 bool ConstWithoutExceptions =
2782 getContext().BuiltinInfo.isConstWithoutExceptions(BuiltinID);
2799 bool ConstWithoutErrnoOrExceptions =
2800 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2801 bool GenerateIntrinsics =
2802 (ConstAlways && !OptNone) ||
2803 (!getLangOpts().MathErrno &&
2804 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2805 if (!GenerateIntrinsics) {
2806 GenerateIntrinsics =
2807 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2808 if (!GenerateIntrinsics)
2809 GenerateIntrinsics =
2810 ConstWithoutErrnoOrExceptions &&
2811 (!getLangOpts().MathErrno &&
2812 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2813 if (!GenerateIntrinsics)
2814 GenerateIntrinsics =
2815 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2817 if (GenerateIntrinsics &&
2818 !(getLangOpts().SYCLIsDevice && getTarget().getTriple().isNVPTX())) {
2819 switch (BuiltinIDIfNoAsmLabel) {
2820 case Builtin::BIceil:
2821 case Builtin::BIceilf:
2822 case Builtin::BIceill:
2823 case Builtin::BI__builtin_ceil:
2824 case Builtin::BI__builtin_ceilf:
2825 case Builtin::BI__builtin_ceilf16:
2826 case Builtin::BI__builtin_ceill:
2827 case Builtin::BI__builtin_ceilf128:
2830 Intrinsic::experimental_constrained_ceil));
2832 case Builtin::BIcopysign:
2833 case Builtin::BIcopysignf:
2834 case Builtin::BIcopysignl:
2835 case Builtin::BI__builtin_copysign:
2836 case Builtin::BI__builtin_copysignf:
2837 case Builtin::BI__builtin_copysignf16:
2838 case Builtin::BI__builtin_copysignl:
2839 case Builtin::BI__builtin_copysignf128:
2842 case Builtin::BIcos:
2843 case Builtin::BIcosf:
2844 case Builtin::BIcosl:
2845 case Builtin::BI__builtin_cos:
2846 case Builtin::BI__builtin_cosf:
2847 case Builtin::BI__builtin_cosf16:
2848 case Builtin::BI__builtin_cosl:
2849 case Builtin::BI__builtin_cosf128:
2851 *
this, E,
Intrinsic::cos, Intrinsic::experimental_constrained_cos,
2852 Intrinsic::fpbuiltin_cos));
2854 case Builtin::BIexp:
2855 case Builtin::BIexpf:
2856 case Builtin::BIexpl:
2857 case Builtin::BI__builtin_exp:
2858 case Builtin::BI__builtin_expf:
2859 case Builtin::BI__builtin_expf16:
2860 case Builtin::BI__builtin_expl:
2861 case Builtin::BI__builtin_expf128:
2863 *
this, E,
Intrinsic::exp, Intrinsic::experimental_constrained_exp,
2864 Intrinsic::fpbuiltin_exp));
2866 case Builtin::BIexp2:
2867 case Builtin::BIexp2f:
2868 case Builtin::BIexp2l:
2869 case Builtin::BI__builtin_exp2:
2870 case Builtin::BI__builtin_exp2f:
2871 case Builtin::BI__builtin_exp2f16:
2872 case Builtin::BI__builtin_exp2l:
2873 case Builtin::BI__builtin_exp2f128:
2876 Intrinsic::fpbuiltin_exp2));
2877 case Builtin::BI__builtin_exp10:
2878 case Builtin::BI__builtin_exp10f:
2879 case Builtin::BI__builtin_exp10f16:
2880 case Builtin::BI__builtin_exp10l:
2881 case Builtin::BI__builtin_exp10f128: {
2883 if (Builder.getIsFPConstrained())
2887 case Builtin::BIfabs:
2888 case Builtin::BIfabsf:
2889 case Builtin::BIfabsl:
2890 case Builtin::BI__builtin_fabs:
2891 case Builtin::BI__builtin_fabsf:
2892 case Builtin::BI__builtin_fabsf16:
2893 case Builtin::BI__builtin_fabsl:
2894 case Builtin::BI__builtin_fabsf128:
2897 case Builtin::BIfloor:
2898 case Builtin::BIfloorf:
2899 case Builtin::BIfloorl:
2900 case Builtin::BI__builtin_floor:
2901 case Builtin::BI__builtin_floorf:
2902 case Builtin::BI__builtin_floorf16:
2903 case Builtin::BI__builtin_floorl:
2904 case Builtin::BI__builtin_floorf128:
2907 Intrinsic::experimental_constrained_floor));
2909 case Builtin::BIfma:
2910 case Builtin::BIfmaf:
2911 case Builtin::BIfmal:
2912 case Builtin::BI__builtin_fma:
2913 case Builtin::BI__builtin_fmaf:
2914 case Builtin::BI__builtin_fmaf16:
2915 case Builtin::BI__builtin_fmal:
2916 case Builtin::BI__builtin_fmaf128:
2919 Intrinsic::experimental_constrained_fma));
2921 case Builtin::BIfmax:
2922 case Builtin::BIfmaxf:
2923 case Builtin::BIfmaxl:
2924 case Builtin::BI__builtin_fmax:
2925 case Builtin::BI__builtin_fmaxf:
2926 case Builtin::BI__builtin_fmaxf16:
2927 case Builtin::BI__builtin_fmaxl:
2928 case Builtin::BI__builtin_fmaxf128:
2931 Intrinsic::experimental_constrained_maxnum));
2933 case Builtin::BIfmin:
2934 case Builtin::BIfminf:
2935 case Builtin::BIfminl:
2936 case Builtin::BI__builtin_fmin:
2937 case Builtin::BI__builtin_fminf:
2938 case Builtin::BI__builtin_fminf16:
2939 case Builtin::BI__builtin_fminl:
2940 case Builtin::BI__builtin_fminf128:
2943 Intrinsic::experimental_constrained_minnum));
2947 case Builtin::BIfmod:
2948 case Builtin::BIfmodf:
2949 case Builtin::BIfmodl:
2950 case Builtin::BI__builtin_fmod:
2951 case Builtin::BI__builtin_fmodf:
2952 case Builtin::BI__builtin_fmodf16:
2953 case Builtin::BI__builtin_fmodl:
2954 case Builtin::BI__builtin_fmodf128: {
2958 return RValue::get(Builder.CreateFRem(Arg1, Arg2,
"fmod"));
2961 case Builtin::BIlog:
2962 case Builtin::BIlogf:
2963 case Builtin::BIlogl:
2964 case Builtin::BI__builtin_log:
2965 case Builtin::BI__builtin_logf:
2966 case Builtin::BI__builtin_logf16:
2967 case Builtin::BI__builtin_logl:
2968 case Builtin::BI__builtin_logf128:
2970 *
this, E,
Intrinsic::log, Intrinsic::experimental_constrained_log,
2971 Intrinsic::fpbuiltin_log));
2973 case Builtin::BIlog10:
2974 case Builtin::BIlog10f:
2975 case Builtin::BIlog10l:
2976 case Builtin::BI__builtin_log10:
2977 case Builtin::BI__builtin_log10f:
2978 case Builtin::BI__builtin_log10f16:
2979 case Builtin::BI__builtin_log10l:
2980 case Builtin::BI__builtin_log10f128:
2983 Intrinsic::fpbuiltin_log10));
2985 case Builtin::BIlog2:
2986 case Builtin::BIlog2f:
2987 case Builtin::BIlog2l:
2988 case Builtin::BI__builtin_log2:
2989 case Builtin::BI__builtin_log2f:
2990 case Builtin::BI__builtin_log2f16:
2991 case Builtin::BI__builtin_log2l:
2992 case Builtin::BI__builtin_log2f128:
2995 Intrinsic::fpbuiltin_log2));
2997 case Builtin::BInearbyint:
2998 case Builtin::BInearbyintf:
2999 case Builtin::BInearbyintl:
3000 case Builtin::BI__builtin_nearbyint:
3001 case Builtin::BI__builtin_nearbyintf:
3002 case Builtin::BI__builtin_nearbyintl:
3003 case Builtin::BI__builtin_nearbyintf128:
3006 Intrinsic::experimental_constrained_nearbyint));
3008 case Builtin::BIpow:
3009 case Builtin::BIpowf:
3010 case Builtin::BIpowl:
3011 case Builtin::BI__builtin_pow:
3012 case Builtin::BI__builtin_powf:
3013 case Builtin::BI__builtin_powf16:
3014 case Builtin::BI__builtin_powl:
3015 case Builtin::BI__builtin_powf128:
3017 *
this, E,
Intrinsic::pow, Intrinsic::experimental_constrained_pow,
3018 Intrinsic::fpbuiltin_pow));
3020 case Builtin::BIrint:
3021 case Builtin::BIrintf:
3022 case Builtin::BIrintl:
3023 case Builtin::BI__builtin_rint:
3024 case Builtin::BI__builtin_rintf:
3025 case Builtin::BI__builtin_rintf16:
3026 case Builtin::BI__builtin_rintl:
3027 case Builtin::BI__builtin_rintf128:
3030 Intrinsic::experimental_constrained_rint));
3032 case Builtin::BIround:
3033 case Builtin::BIroundf:
3034 case Builtin::BIroundl:
3035 case Builtin::BI__builtin_round:
3036 case Builtin::BI__builtin_roundf:
3037 case Builtin::BI__builtin_roundf16:
3038 case Builtin::BI__builtin_roundl:
3039 case Builtin::BI__builtin_roundf128:
3042 Intrinsic::experimental_constrained_round));
3044 case Builtin::BIroundeven:
3045 case Builtin::BIroundevenf:
3046 case Builtin::BIroundevenl:
3047 case Builtin::BI__builtin_roundeven:
3048 case Builtin::BI__builtin_roundevenf:
3049 case Builtin::BI__builtin_roundevenf16:
3050 case Builtin::BI__builtin_roundevenl:
3051 case Builtin::BI__builtin_roundevenf128:
3053 Intrinsic::roundeven,
3054 Intrinsic::experimental_constrained_roundeven));
3056 case Builtin::BIsin:
3057 case Builtin::BIsinf:
3058 case Builtin::BIsinl:
3059 case Builtin::BI__builtin_sin:
3060 case Builtin::BI__builtin_sinf:
3061 case Builtin::BI__builtin_sinf16:
3062 case Builtin::BI__builtin_sinl:
3063 case Builtin::BI__builtin_sinf128:
3065 *
this, E,
Intrinsic::sin, Intrinsic::experimental_constrained_sin,
3066 Intrinsic::fpbuiltin_sin));
3068 case Builtin::BIsqrt:
3069 case Builtin::BIsqrtf:
3070 case Builtin::BIsqrtl:
3071 case Builtin::BI__builtin_sqrt:
3072 case Builtin::BI__builtin_sqrtf:
3073 case Builtin::BI__builtin_sqrtf16:
3074 case Builtin::BI__builtin_sqrtl:
3075 case Builtin::BI__builtin_sqrtf128:
3076 case Builtin::BI__builtin_elementwise_sqrt: {
3078 *
this, E,
Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt, Intrinsic::fpbuiltin_sqrt);
3079 SetSqrtFPAccuracy(Call);
3082 case Builtin::BItrunc:
3083 case Builtin::BItruncf:
3084 case Builtin::BItruncl:
3085 case Builtin::BI__builtin_trunc:
3086 case Builtin::BI__builtin_truncf:
3087 case Builtin::BI__builtin_truncf16:
3088 case Builtin::BI__builtin_truncl:
3089 case Builtin::BI__builtin_truncf128:
3092 Intrinsic::experimental_constrained_trunc));
3094 case Builtin::BIlround:
3095 case Builtin::BIlroundf:
3096 case Builtin::BIlroundl:
3097 case Builtin::BI__builtin_lround:
3098 case Builtin::BI__builtin_lroundf:
3099 case Builtin::BI__builtin_lroundl:
3100 case Builtin::BI__builtin_lroundf128:
3103 Intrinsic::experimental_constrained_lround));
3105 case Builtin::BIllround:
3106 case Builtin::BIllroundf:
3107 case Builtin::BIllroundl:
3108 case Builtin::BI__builtin_llround:
3109 case Builtin::BI__builtin_llroundf:
3110 case Builtin::BI__builtin_llroundl:
3111 case Builtin::BI__builtin_llroundf128:
3114 Intrinsic::experimental_constrained_llround));
3116 case Builtin::BIlrint:
3117 case Builtin::BIlrintf:
3118 case Builtin::BIlrintl:
3119 case Builtin::BI__builtin_lrint:
3120 case Builtin::BI__builtin_lrintf:
3121 case Builtin::BI__builtin_lrintl:
3122 case Builtin::BI__builtin_lrintf128:
3125 Intrinsic::experimental_constrained_lrint));
3127 case Builtin::BIllrint:
3128 case Builtin::BIllrintf:
3129 case Builtin::BIllrintl:
3130 case Builtin::BI__builtin_llrint:
3131 case Builtin::BI__builtin_llrintf:
3132 case Builtin::BI__builtin_llrintl:
3133 case Builtin::BI__builtin_llrintf128:
3136 Intrinsic::experimental_constrained_llrint));
3137 case Builtin::BI__builtin_ldexp:
3138 case Builtin::BI__builtin_ldexpf:
3139 case Builtin::BI__builtin_ldexpl:
3140 case Builtin::BI__builtin_ldexpf16:
3141 case Builtin::BI__builtin_ldexpf128: {
3144 Intrinsic::experimental_constrained_ldexp));
3154 Value *Val = A.emitRawPointer(*
this);
3155 EmitNonNullArgCheck(
RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD,
3158 if (SanOpts.has(SanitizerKind::Alignment)) {
3161 SkippedChecks.
clear(SanitizerKind::Alignment);
3164 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3165 if (CE->getCastKind() == CK_BitCast)
3166 Arg = CE->getSubExpr();
3167 EmitTypeCheck(
Kind, Loc, Val, Arg->
getType(), A.getAlignment(),
3172 switch (BuiltinIDIfNoAsmLabel) {
3174 case Builtin::BI__builtin___CFStringMakeConstantString:
3175 case Builtin::BI__builtin___NSStringMakeConstantString:
3177 case Builtin::BI__builtin_stdarg_start:
3178 case Builtin::BI__builtin_va_start:
3179 case Builtin::BI__va_start:
3180 case Builtin::BI__builtin_va_end:
3181 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
3182 ? EmitScalarExpr(E->
getArg(0))
3183 : EmitVAListRef(E->
getArg(0)).emitRawPointer(*
this),
3184 BuiltinID != Builtin::BI__builtin_va_end);
3186 case Builtin::BI__builtin_va_copy: {
3187 Value *DstPtr = EmitVAListRef(E->
getArg(0)).emitRawPointer(*
this);
3188 Value *SrcPtr = EmitVAListRef(E->
getArg(1)).emitRawPointer(*
this);
3189 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}),
3193 case Builtin::BIabs:
3194 case Builtin::BIlabs:
3195 case Builtin::BIllabs:
3196 case Builtin::BI__builtin_abs:
3197 case Builtin::BI__builtin_labs:
3198 case Builtin::BI__builtin_llabs: {
3199 bool SanitizeOverflow = SanOpts.has(SanitizerKind::SignedIntegerOverflow);
3202 switch (getLangOpts().getSignedOverflowBehavior()) {
3204 Result =
EmitAbs(*
this, EmitScalarExpr(E->
getArg(0)),
false);
3207 if (!SanitizeOverflow) {
3208 Result =
EmitAbs(*
this, EmitScalarExpr(E->
getArg(0)),
true);
3219 case Builtin::BI__builtin_complex: {
3224 case Builtin::BI__builtin_conj:
3225 case Builtin::BI__builtin_conjf:
3226 case Builtin::BI__builtin_conjl:
3227 case Builtin::BIconj:
3228 case Builtin::BIconjf:
3229 case Builtin::BIconjl: {
3231 Value *Real = ComplexVal.first;
3232 Value *Imag = ComplexVal.second;
3233 Imag = Builder.CreateFNeg(Imag,
"neg");
3236 case Builtin::BI__builtin_creal:
3237 case Builtin::BI__builtin_crealf:
3238 case Builtin::BI__builtin_creall:
3239 case Builtin::BIcreal:
3240 case Builtin::BIcrealf:
3241 case Builtin::BIcreall: {
3246 case Builtin::BI__builtin_preserve_access_index: {
3250 if (!getDebugInfo()) {
3251 CGM.Error(E->
getExprLoc(),
"using builtin_preserve_access_index() without -g");
3256 if (IsInPreservedAIRegion) {
3257 CGM.Error(E->
getExprLoc(),
"nested builtin_preserve_access_index() not supported");
3261 IsInPreservedAIRegion =
true;
3263 IsInPreservedAIRegion =
false;
3267 case Builtin::BI__builtin_cimag:
3268 case Builtin::BI__builtin_cimagf:
3269 case Builtin::BI__builtin_cimagl:
3270 case Builtin::BIcimag:
3271 case Builtin::BIcimagf:
3272 case Builtin::BIcimagl: {
3277 case Builtin::BI__builtin_clrsb:
3278 case Builtin::BI__builtin_clrsbl:
3279 case Builtin::BI__builtin_clrsbll: {
3283 llvm::Type *ArgType = ArgValue->
getType();
3284 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3286 llvm::Type *ResultType = ConvertType(E->
getType());
3287 Value *Zero = llvm::Constant::getNullValue(ArgType);
3288 Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3289 Value *Inverse = Builder.CreateNot(ArgValue,
"not");
3290 Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3291 Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
3292 Value *Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
3293 Result = Builder.CreateIntCast(Result, ResultType,
true,
3297 case Builtin::BI__builtin_ctzs:
3298 case Builtin::BI__builtin_ctz:
3299 case Builtin::BI__builtin_ctzl:
3300 case Builtin::BI__builtin_ctzll:
3301 case Builtin::BI__builtin_ctzg: {
3302 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3306 HasFallback ? EmitScalarExpr(E->
getArg(0))
3307 : EmitCheckedArgForBuiltin(E->
getArg(0), BCK_CTZPassedZero);
3309 llvm::Type *ArgType = ArgValue->
getType();
3310 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3312 llvm::Type *ResultType = ConvertType(E->
getType());
3314 Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
3315 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3316 if (Result->getType() != ResultType)
3317 Result = Builder.CreateIntCast(Result, ResultType,
true,
3322 Value *Zero = Constant::getNullValue(ArgType);
3323 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3324 Value *FallbackValue = EmitScalarExpr(E->
getArg(1));
3325 Value *ResultOrFallback =
3326 Builder.CreateSelect(IsZero, FallbackValue, Result,
"ctzg");
3329 case Builtin::BI__builtin_clzs:
3330 case Builtin::BI__builtin_clz:
3331 case Builtin::BI__builtin_clzl:
3332 case Builtin::BI__builtin_clzll:
3333 case Builtin::BI__builtin_clzg: {
3334 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3338 HasFallback ? EmitScalarExpr(E->
getArg(0))
3339 : EmitCheckedArgForBuiltin(E->
getArg(0), BCK_CLZPassedZero);
3341 llvm::Type *ArgType = ArgValue->
getType();
3342 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3344 llvm::Type *ResultType = ConvertType(E->
getType());
3346 Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
3347 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3348 if (Result->getType() != ResultType)
3349 Result = Builder.CreateIntCast(Result, ResultType,
true,
3354 Value *Zero = Constant::getNullValue(ArgType);
3355 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3356 Value *FallbackValue = EmitScalarExpr(E->
getArg(1));
3357 Value *ResultOrFallback =
3358 Builder.CreateSelect(IsZero, FallbackValue, Result,
"clzg");
3361 case Builtin::BI__builtin_ffs:
3362 case Builtin::BI__builtin_ffsl:
3363 case Builtin::BI__builtin_ffsll: {
3367 llvm::Type *ArgType = ArgValue->
getType();
3368 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3370 llvm::Type *ResultType = ConvertType(E->
getType());
3372 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3373 llvm::ConstantInt::get(ArgType, 1));
3374 Value *Zero = llvm::Constant::getNullValue(ArgType);
3375 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3376 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp,
"ffs");
3377 if (Result->getType() != ResultType)
3378 Result = Builder.CreateIntCast(Result, ResultType,
true,
3382 case Builtin::BI__builtin_parity:
3383 case Builtin::BI__builtin_parityl:
3384 case Builtin::BI__builtin_parityll: {
3388 llvm::Type *ArgType = ArgValue->
getType();
3389 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3391 llvm::Type *ResultType = ConvertType(E->
getType());
3392 Value *Tmp = Builder.CreateCall(F, ArgValue);
3393 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
3394 if (Result->getType() != ResultType)
3395 Result = Builder.CreateIntCast(Result, ResultType,
true,
3399 case Builtin::BI__lzcnt16:
3400 case Builtin::BI__lzcnt:
3401 case Builtin::BI__lzcnt64: {
3404 llvm::Type *ArgType = ArgValue->
getType();
3405 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3407 llvm::Type *ResultType = ConvertType(E->
getType());
3408 Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
3409 if (Result->getType() != ResultType)
3410 Result = Builder.CreateIntCast(Result, ResultType,
true,
3414 case Builtin::BI__popcnt16:
3415 case Builtin::BI__popcnt:
3416 case Builtin::BI__popcnt64:
3417 case Builtin::BI__builtin_popcount:
3418 case Builtin::BI__builtin_popcountl:
3419 case Builtin::BI__builtin_popcountll:
3420 case Builtin::BI__builtin_popcountg: {
3423 llvm::Type *ArgType = ArgValue->
getType();
3424 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3426 llvm::Type *ResultType = ConvertType(E->
getType());
3427 Value *Result = Builder.CreateCall(F, ArgValue);
3428 if (Result->getType() != ResultType)
3429 Result = Builder.CreateIntCast(Result, ResultType,
true,
3433 case Builtin::BI__builtin_unpredictable: {
3439 case Builtin::BI__builtin_expect: {
3441 llvm::Type *ArgType = ArgValue->
getType();
3443 Value *ExpectedValue = EmitScalarExpr(E->
getArg(1));
3447 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3450 Function *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
3452 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3455 case Builtin::BI__builtin_expect_with_probability: {
3457 llvm::Type *ArgType = ArgValue->
getType();
3459 Value *ExpectedValue = EmitScalarExpr(E->
getArg(1));
3462 bool EvalSucceed = ProbArg->
EvaluateAsFloat(Probability, CGM.getContext());
3463 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3465 bool LoseInfo =
false;
3466 Probability.convert(llvm::APFloat::IEEEdouble(),
3467 llvm::RoundingMode::Dynamic, &LoseInfo);
3468 llvm::Type *Ty = ConvertType(ProbArg->
getType());
3469 Constant *Confidence = ConstantFP::get(Ty, Probability);
3473 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3477 CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
3478 Value *Result = Builder.CreateCall(
3479 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3482 case Builtin::BI__builtin_assume_aligned: {
3484 Value *PtrValue = EmitScalarExpr(Ptr);
3485 Value *OffsetValue =
3488 Value *AlignmentValue = EmitScalarExpr(E->
getArg(1));
3489 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3490 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3491 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3492 llvm::Value::MaximumAlignment);
3494 emitAlignmentAssumption(PtrValue, Ptr,
3496 AlignmentCI, OffsetValue);
3499 case Builtin::BI__assume:
3500 case Builtin::BI__builtin_assume: {
3505 Function *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
3506 Builder.CreateCall(FnAssume, ArgValue);
3509 case Builtin::BI__builtin_assume_separate_storage: {
3513 Value *Value0 = EmitScalarExpr(Arg0);
3514 Value *Value1 = EmitScalarExpr(Arg1);
3516 Value *Values[] = {Value0, Value1};
3517 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3518 Builder.CreateAssumption(ConstantInt::getTrue(getLLVMContext()), {OBD});
3521 case Builtin::BI__builtin_allow_runtime_check: {
3524 LLVMContext &Ctx = CGM.getLLVMContext();
3525 llvm::Value *Allow = Builder.CreateCall(
3526 CGM.getIntrinsic(llvm::Intrinsic::allow_runtime_check),
3527 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx,
Kind)));
3530 case Builtin::BI__arithmetic_fence: {
3534 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3535 bool isArithmeticFenceEnabled =
3536 FMF.allowReassoc() &&
3537 getContext().getTargetInfo().checkArithmeticFenceSupported();
3540 if (isArithmeticFenceEnabled) {
3543 Value *Real = Builder.CreateArithmeticFence(ComplexVal.first,
3544 ConvertType(ElementType));
3545 Value *Imag = Builder.CreateArithmeticFence(ComplexVal.second,
3546 ConvertType(ElementType));
3550 Value *Real = ComplexVal.first;
3551 Value *Imag = ComplexVal.second;
3555 if (isArithmeticFenceEnabled)
3557 Builder.CreateArithmeticFence(ArgValue, ConvertType(ArgType)));
3560 case Builtin::BI__builtin_bswap16:
3561 case Builtin::BI__builtin_bswap32:
3562 case Builtin::BI__builtin_bswap64:
3563 case Builtin::BI_byteswap_ushort:
3564 case Builtin::BI_byteswap_ulong:
3565 case Builtin::BI_byteswap_uint64: {
3568 case Builtin::BI__builtin_bitreverse8:
3569 case Builtin::BI__builtin_bitreverse16:
3570 case Builtin::BI__builtin_bitreverse32:
3571 case Builtin::BI__builtin_bitreverse64: {
3574 case Builtin::BI__builtin_rotateleft8:
3575 case Builtin::BI__builtin_rotateleft16:
3576 case Builtin::BI__builtin_rotateleft32:
3577 case Builtin::BI__builtin_rotateleft64:
3578 case Builtin::BI_rotl8:
3579 case Builtin::BI_rotl16:
3580 case Builtin::BI_rotl:
3581 case Builtin::BI_lrotl:
3582 case Builtin::BI_rotl64:
3583 return emitRotate(E,
false);
3585 case Builtin::BI__builtin_rotateright8:
3586 case Builtin::BI__builtin_rotateright16:
3587 case Builtin::BI__builtin_rotateright32:
3588 case Builtin::BI__builtin_rotateright64:
3589 case Builtin::BI_rotr8:
3590 case Builtin::BI_rotr16:
3591 case Builtin::BI_rotr:
3592 case Builtin::BI_lrotr:
3593 case Builtin::BI_rotr64:
3594 return emitRotate(E,
true);
3596 case Builtin::BI__builtin_constant_p: {
3597 llvm::Type *ResultType = ConvertType(E->
getType());
3607 return RValue::get(ConstantInt::get(ResultType, 0));
3612 return RValue::get(ConstantInt::get(ResultType, 0));
3614 Value *ArgValue = EmitScalarExpr(Arg);
3618 ArgType = CGM.getContext().getObjCIdType();
3619 ArgValue = Builder.CreateBitCast(ArgValue, ConvertType(ArgType));
3622 CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
3623 Value *Result = Builder.CreateCall(F, ArgValue);
3624 if (Result->getType() != ResultType)
3625 Result = Builder.CreateIntCast(Result, ResultType,
false);
3628 case Builtin::BI__builtin_dynamic_object_size:
3629 case Builtin::BI__builtin_object_size: {
3632 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->
getType()));
3636 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3638 nullptr, IsDynamic));
3640 case Builtin::BI__builtin_prefetch: {
3644 llvm::ConstantInt::get(Int32Ty, 0);
3646 llvm::ConstantInt::get(Int32Ty, 3);
3647 Value *
Data = llvm::ConstantInt::get(Int32Ty, 1);
3649 Builder.CreateCall(F, {
Address, RW, Locality,
Data});
3652 case Builtin::BI__builtin_readcyclecounter: {
3653 Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
3656 case Builtin::BI__builtin_readsteadycounter: {
3657 Function *F = CGM.getIntrinsic(Intrinsic::readsteadycounter);
3660 case Builtin::BI__builtin___clear_cache: {
3663 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache);
3664 return RValue::get(Builder.CreateCall(F, {Begin, End}));
3666 case Builtin::BI__builtin_trap:
3667 EmitTrapCall(Intrinsic::trap);
3669 case Builtin::BI__debugbreak:
3670 EmitTrapCall(Intrinsic::debugtrap);
3672 case Builtin::BI__builtin_unreachable: {
3676 EmitBlock(createBasicBlock(
"unreachable.cont"));
3681 case Builtin::BI__builtin_powi:
3682 case Builtin::BI__builtin_powif:
3683 case Builtin::BI__builtin_powil: {
3684 llvm::Value *Src0 = EmitScalarExpr(E->
getArg(0));
3685 llvm::Value *Src1 = EmitScalarExpr(E->
getArg(1));
3687 if (Builder.getIsFPConstrained()) {
3691 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_powi,
3693 return RValue::get(Builder.CreateConstrainedFPCall(F, { Src0, Src1 }));
3697 { Src0->getType(), Src1->getType() });
3698 return RValue::get(Builder.CreateCall(F, { Src0, Src1 }));
3700 case Builtin::BI__builtin_frexpl: {
3704 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3708 case Builtin::BI__builtin_frexp:
3709 case Builtin::BI__builtin_frexpf:
3710 case Builtin::BI__builtin_frexpf128:
3711 case Builtin::BI__builtin_frexpf16:
3713 case Builtin::BI__builtin_isgreater:
3714 case Builtin::BI__builtin_isgreaterequal:
3715 case Builtin::BI__builtin_isless:
3716 case Builtin::BI__builtin_islessequal:
3717 case Builtin::BI__builtin_islessgreater:
3718 case Builtin::BI__builtin_isunordered: {
3725 switch (BuiltinID) {
3726 default: llvm_unreachable(
"Unknown ordered comparison");
3727 case Builtin::BI__builtin_isgreater:
3728 LHS = Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3730 case Builtin::BI__builtin_isgreaterequal:
3731 LHS = Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3733 case Builtin::BI__builtin_isless:
3734 LHS = Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3736 case Builtin::BI__builtin_islessequal:
3737 LHS = Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
3739 case Builtin::BI__builtin_islessgreater:
3740 LHS = Builder.CreateFCmpONE(LHS, RHS,
"cmp");
3742 case Builtin::BI__builtin_isunordered:
3743 LHS = Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3750 case Builtin::BI__builtin_isnan: {
3756 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcNan),
3760 case Builtin::BI__builtin_issignaling: {
3764 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcSNan),
3768 case Builtin::BI__builtin_isinf: {
3774 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcInf),
3778 case Builtin::BIfinite:
3779 case Builtin::BI__finite:
3780 case Builtin::BIfinitef:
3781 case Builtin::BI__finitef:
3782 case Builtin::BIfinitel:
3783 case Builtin::BI__finitel:
3784 case Builtin::BI__builtin_isfinite: {
3790 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcFinite),
3794 case Builtin::BI__builtin_isnormal: {
3798 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcNormal),
3802 case Builtin::BI__builtin_issubnormal: {
3806 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
3810 case Builtin::BI__builtin_iszero: {
3814 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcZero),
3818 case Builtin::BI__builtin_isfpclass: {
3822 uint64_t Test = Result.Val.getInt().getLimitedValue();
3825 return RValue::get(Builder.CreateZExt(Builder.createIsFPClass(
V, Test),
3829 case Builtin::BI__builtin_nondeterministic_value: {
3832 Value *Result = PoisonValue::get(Ty);
3833 Result = Builder.CreateFreeze(Result);
3838 case Builtin::BI__builtin_elementwise_abs: {
3843 QT = VecTy->getElementType();
3845 Result = Builder.CreateBinaryIntrinsic(
3847 Builder.getFalse(),
nullptr,
"elt.abs");
3854 case Builtin::BI__builtin_elementwise_ceil:
3857 case Builtin::BI__builtin_elementwise_exp:
3860 case Builtin::BI__builtin_elementwise_exp2:
3863 case Builtin::BI__builtin_elementwise_log:
3866 case Builtin::BI__builtin_elementwise_log2:
3869 case Builtin::BI__builtin_elementwise_log10:
3872 case Builtin::BI__builtin_elementwise_pow: {
3875 case Builtin::BI__builtin_elementwise_bitreverse:
3878 case Builtin::BI__builtin_elementwise_cos:
3881 case Builtin::BI__builtin_elementwise_floor:
3884 case Builtin::BI__builtin_elementwise_roundeven:
3887 case Builtin::BI__builtin_elementwise_round:
3890 case Builtin::BI__builtin_elementwise_rint:
3893 case Builtin::BI__builtin_elementwise_nearbyint:
3896 case Builtin::BI__builtin_elementwise_sin:
3900 case Builtin::BI__builtin_elementwise_trunc:
3903 case Builtin::BI__builtin_elementwise_canonicalize:
3905 emitUnaryBuiltin(*
this, E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
3906 case Builtin::BI__builtin_elementwise_copysign:
3908 case Builtin::BI__builtin_elementwise_fma:
3910 case Builtin::BI__builtin_elementwise_add_sat:
3911 case Builtin::BI__builtin_elementwise_sub_sat: {
3915 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
3918 Ty = VecTy->getElementType();
3921 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3922 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3924 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3925 Result = Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3929 case Builtin::BI__builtin_elementwise_max: {
3933 if (Op0->
getType()->isIntOrIntVectorTy()) {
3936 Ty = VecTy->getElementType();
3938 ? llvm::Intrinsic::smax
3940 Op0, Op1,
nullptr,
"elt.max");
3942 Result = Builder.CreateMaxNum(Op0, Op1,
"elt.max");
3945 case Builtin::BI__builtin_elementwise_min: {
3949 if (Op0->
getType()->isIntOrIntVectorTy()) {
3952 Ty = VecTy->getElementType();
3954 ? llvm::Intrinsic::smin
3956 Op0, Op1,
nullptr,
"elt.min");
3958 Result = Builder.CreateMinNum(Op0, Op1,
"elt.min");
3962 case Builtin::BI__builtin_reduce_max: {
3963 auto GetIntrinsicID = [](
QualType QT) {
3965 QT = VecTy->getElementType();
3966 if (QT->isSignedIntegerType())
3967 return llvm::Intrinsic::vector_reduce_smax;
3968 if (QT->isUnsignedIntegerType())
3969 return llvm::Intrinsic::vector_reduce_umax;
3970 assert(QT->isFloatingType() &&
"must have a float here");
3971 return llvm::Intrinsic::vector_reduce_fmax;
3974 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3977 case Builtin::BI__builtin_reduce_min: {
3978 auto GetIntrinsicID = [](
QualType QT) {
3980 QT = VecTy->getElementType();
3981 if (QT->isSignedIntegerType())
3982 return llvm::Intrinsic::vector_reduce_smin;
3983 if (QT->isUnsignedIntegerType())
3984 return llvm::Intrinsic::vector_reduce_umin;
3985 assert(QT->isFloatingType() &&
"must have a float here");
3986 return llvm::Intrinsic::vector_reduce_fmin;
3990 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3993 case Builtin::BI__builtin_reduce_add:
3995 *
this, E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3996 case Builtin::BI__builtin_reduce_mul:
3998 *
this, E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3999 case Builtin::BI__builtin_reduce_xor:
4001 *
this, E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
4002 case Builtin::BI__builtin_reduce_or:
4004 *
this, E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
4005 case Builtin::BI__builtin_reduce_and:
4007 *
this, E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
4009 case Builtin::BI__builtin_matrix_transpose: {
4012 MatrixBuilder MB(Builder);
4013 Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
4014 MatrixTy->getNumColumns());
4018 case Builtin::BI__builtin_matrix_column_major_load: {
4019 MatrixBuilder MB(Builder);
4024 assert(PtrTy &&
"arg0 must be of pointer type");
4031 Value *Result = MB.CreateColumnMajorLoad(
4034 ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
4038 case Builtin::BI__builtin_matrix_column_major_store: {
4039 MatrixBuilder MB(Builder);
4046 assert(PtrTy &&
"arg1 must be of pointer type");
4052 Value *Result = MB.CreateColumnMajorStore(
4055 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
4059 case Builtin::BI__builtin_isinf_sign: {
4065 Value *IsInf = Builder.CreateFCmpOEQ(
4066 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
4069 llvm::Type *IntTy = ConvertType(E->
getType());
4070 Value *Zero = Constant::getNullValue(IntTy);
4071 Value *One = ConstantInt::get(IntTy, 1);
4072 Value *NegativeOne = ConstantInt::get(IntTy, -1);
4073 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
4074 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
4078 case Builtin::BI__builtin_flt_rounds: {
4079 Function *F = CGM.getIntrinsic(Intrinsic::get_rounding);
4081 llvm::Type *ResultType = ConvertType(E->
getType());
4082 Value *Result = Builder.CreateCall(F);
4083 if (Result->getType() != ResultType)
4084 Result = Builder.CreateIntCast(Result, ResultType,
true,
4089 case Builtin::BI__builtin_set_flt_rounds: {
4090 Function *F = CGM.getIntrinsic(Intrinsic::set_rounding);
4093 Builder.CreateCall(F,
V);
4097 case Builtin::BI__builtin_fpclassify: {
4104 BasicBlock *
Begin = Builder.GetInsertBlock();
4105 BasicBlock *
End = createBasicBlock(
"fpclassify_end", this->CurFn);
4106 Builder.SetInsertPoint(
End);
4109 "fpclassify_result");
4112 Builder.SetInsertPoint(
Begin);
4113 Value *IsZero = Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
4115 Value *ZeroLiteral = EmitScalarExpr(E->
getArg(4));
4116 BasicBlock *NotZero = createBasicBlock(
"fpclassify_not_zero", this->CurFn);
4117 Builder.CreateCondBr(IsZero,
End, NotZero);
4118 Result->addIncoming(ZeroLiteral,
Begin);
4121 Builder.SetInsertPoint(NotZero);
4122 Value *IsNan = Builder.CreateFCmpUNO(
V,
V,
"cmp");
4124 BasicBlock *NotNan = createBasicBlock(
"fpclassify_not_nan", this->CurFn);
4125 Builder.CreateCondBr(IsNan,
End, NotNan);
4126 Result->addIncoming(NanLiteral, NotZero);
4129 Builder.SetInsertPoint(NotNan);
4132 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
4135 BasicBlock *NotInf = createBasicBlock(
"fpclassify_not_inf", this->CurFn);
4136 Builder.CreateCondBr(IsInf,
End, NotInf);
4137 Result->addIncoming(InfLiteral, NotNan);
4140 Builder.SetInsertPoint(NotInf);
4141 APFloat Smallest = APFloat::getSmallestNormalized(
4142 getContext().getFloatTypeSemantics(E->
getArg(5)->
getType()));
4144 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
4146 Value *NormalResult =
4147 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->
getArg(2)),
4148 EmitScalarExpr(E->
getArg(3)));
4149 Builder.CreateBr(
End);
4150 Result->addIncoming(NormalResult, NotInf);
4153 Builder.SetInsertPoint(
End);
4163 case Builtin::BIalloca:
4164 case Builtin::BI_alloca:
4165 case Builtin::BI__builtin_alloca_uninitialized:
4166 case Builtin::BI__builtin_alloca: {
4168 const TargetInfo &TI = getContext().getTargetInfo();
4170 const Align SuitableAlignmentInBytes =
4174 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
4175 AI->setAlignment(SuitableAlignmentInBytes);
4176 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4178 LangAS AAS = getASTAllocaAddressSpace();
4181 llvm::Type *Ty = CGM.getTypes().ConvertType(E->
getType());
4182 return RValue::get(getTargetHooks().performAddrSpaceCast(*
this, AI, AAS,
4188 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4189 case Builtin::BI__builtin_alloca_with_align: {
4191 Value *AlignmentInBitsValue = EmitScalarExpr(E->
getArg(1));
4192 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4193 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4194 const Align AlignmentInBytes =
4195 CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getAsAlign();
4196 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
4197 AI->setAlignment(AlignmentInBytes);
4198 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4200 LangAS AAS = getASTAllocaAddressSpace();
4203 llvm::Type *Ty = CGM.getTypes().ConvertType(E->
getType());
4204 return RValue::get(getTargetHooks().performAddrSpaceCast(*
this, AI, AAS,
4210 case Builtin::BIbzero:
4211 case Builtin::BI__builtin_bzero: {
4216 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal,
false);
4220 case Builtin::BIbcopy:
4221 case Builtin::BI__builtin_bcopy: {
4231 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
4235 case Builtin::BImemcpy:
4236 case Builtin::BI__builtin_memcpy:
4237 case Builtin::BImempcpy:
4238 case Builtin::BI__builtin_mempcpy: {
4242 EmitArgCheck(TCK_Store, Dest, E->
getArg(0), 0);
4243 EmitArgCheck(TCK_Load, Src, E->
getArg(1), 1);
4244 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
4245 if (BuiltinID == Builtin::BImempcpy ||
4246 BuiltinID == Builtin::BI__builtin_mempcpy)
4253 case Builtin::BI__builtin_memcpy_inline: {
4258 EmitArgCheck(TCK_Store, Dest, E->
getArg(0), 0);
4259 EmitArgCheck(TCK_Load, Src, E->
getArg(1), 1);
4260 Builder.CreateMemCpyInline(Dest, Src, Size);
4264 case Builtin::BI__builtin_char_memchr:
4265 BuiltinID = Builtin::BI__builtin_memchr;
4268 case Builtin::BI__builtin___memcpy_chk: {
4276 if (Size.ugt(DstSize))
4280 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4281 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
4285 case Builtin::BI__builtin_objc_memmove_collectable: {
4289 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*
this,
4290 DestAddr, SrcAddr, SizeVal);
4294 case Builtin::BI__builtin___memmove_chk: {
4302 if (Size.ugt(DstSize))
4306 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4307 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
4311 case Builtin::BImemmove:
4312 case Builtin::BI__builtin_memmove: {
4316 EmitArgCheck(TCK_Store, Dest, E->
getArg(0), 0);
4317 EmitArgCheck(TCK_Load, Src, E->
getArg(1), 1);
4318 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
4321 case Builtin::BImemset:
4322 case Builtin::BI__builtin_memset: {
4324 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->
getArg(1)),
4325 Builder.getInt8Ty());
4329 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
4332 case Builtin::BI__builtin_memset_inline: {
4335 Builder.CreateTrunc(EmitScalarExpr(E->
getArg(1)), Builder.getInt8Ty());
4341 Builder.CreateMemSetInline(Dest, ByteVal, Size);
4344 case Builtin::BI__builtin___memset_chk: {
4352 if (Size.ugt(DstSize))
4355 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->
getArg(1)),
4356 Builder.getInt8Ty());
4357 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4358 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
4361 case Builtin::BI__builtin_wmemchr: {
4364 if (!getTarget().getTriple().isOSMSVCRT())
4367 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
4372 BasicBlock *Entry = Builder.GetInsertBlock();
4373 BasicBlock *CmpEq = createBasicBlock(
"wmemchr.eq");
4374 BasicBlock *Next = createBasicBlock(
"wmemchr.next");
4375 BasicBlock *Exit = createBasicBlock(
"wmemchr.exit");
4376 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
4377 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4380 PHINode *StrPhi = Builder.CreatePHI(Str->
getType(), 2);
4381 StrPhi->addIncoming(Str, Entry);
4382 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
4383 SizePhi->addIncoming(Size, Entry);
4385 getContext().getTypeAlignInChars(getContext().WCharTy);
4386 Value *StrCh = Builder.CreateAlignedLoad(WCharTy, StrPhi, WCharAlign);
4387 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4388 Value *StrEqChr = Builder.CreateICmpEQ(StrCh, Chr);
4389 Builder.CreateCondBr(StrEqChr, Exit, Next);
4392 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4393 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
4394 Value *NextSizeEq0 =
4395 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
4396 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4397 StrPhi->addIncoming(NextStr, Next);
4398 SizePhi->addIncoming(NextSize, Next);
4401 PHINode *
Ret = Builder.CreatePHI(Str->
getType(), 3);
4402 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4403 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4404 Ret->addIncoming(FoundChr, CmpEq);
4407 case Builtin::BI__builtin_wmemcmp: {
4410 if (!getTarget().getTriple().isOSMSVCRT())
4413 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
4419 BasicBlock *Entry = Builder.GetInsertBlock();
4420 BasicBlock *CmpGT = createBasicBlock(
"wmemcmp.gt");
4421 BasicBlock *CmpLT = createBasicBlock(
"wmemcmp.lt");
4422 BasicBlock *Next = createBasicBlock(
"wmemcmp.next");
4423 BasicBlock *Exit = createBasicBlock(
"wmemcmp.exit");
4424 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
4425 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4428 PHINode *DstPhi = Builder.CreatePHI(Dst->
getType(), 2);
4429 DstPhi->addIncoming(Dst, Entry);
4430 PHINode *SrcPhi = Builder.CreatePHI(Src->
getType(), 2);
4431 SrcPhi->addIncoming(Src, Entry);
4432 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
4433 SizePhi->addIncoming(Size, Entry);
4435 getContext().getTypeAlignInChars(getContext().WCharTy);
4436 Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
4437 Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
4438 Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
4439 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4442 Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
4443 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4446 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4447 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4448 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
4449 Value *NextSizeEq0 =
4450 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
4451 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4452 DstPhi->addIncoming(NextDst, Next);
4453 SrcPhi->addIncoming(NextSrc, Next);
4454 SizePhi->addIncoming(NextSize, Next);
4457 PHINode *
Ret = Builder.CreatePHI(IntTy, 4);
4458 Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
4459 Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
4460 Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
4461 Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
4464 case Builtin::BI__builtin_dwarf_cfa: {
4475 Function *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
4477 llvm::ConstantInt::get(Int32Ty,
Offset)));
4479 case Builtin::BI__builtin_return_address: {
4481 getContext().UnsignedIntTy);
4482 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4485 case Builtin::BI_ReturnAddress: {
4486 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4487 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
4489 case Builtin::BI__builtin_frame_address: {
4491 getContext().UnsignedIntTy);
4492 Function *F = CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy);
4495 case Builtin::BI__builtin_extract_return_addr: {
4497 Value *Result = getTargetHooks().decodeReturnAddress(*
this,
Address);
4500 case Builtin::BI__builtin_frob_return_addr: {
4502 Value *Result = getTargetHooks().encodeReturnAddress(*
this,
Address);
4505 case Builtin::BI__builtin_dwarf_sp_column: {
4506 llvm::IntegerType *Ty
4507 = cast<llvm::IntegerType>(ConvertType(E->
getType()));
4508 int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
4510 CGM.ErrorUnsupported(E,
"__builtin_dwarf_sp_column");
4515 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4517 if (getTargetHooks().initDwarfEHRegSizeTable(*
this,
Address))
4518 CGM.ErrorUnsupported(E,
"__builtin_init_dwarf_reg_size_table");
4521 case Builtin::BI__builtin_eh_return: {
4525 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
4526 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
4527 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4529 CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
4530 : Intrinsic::eh_return_i64);
4531 Builder.CreateCall(F, {Int, Ptr});
4532 Builder.CreateUnreachable();
4535 EmitBlock(createBasicBlock(
"builtin_eh_return.cont"));
4539 case Builtin::BI__builtin_unwind_init: {
4540 Function *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
4541 Builder.CreateCall(F);
4544 case Builtin::BI__builtin_extend_pointer: {
4557 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy,
"extend.cast");
4560 if (IntPtrTy->getBitWidth() == 64)
4564 if (getTargetHooks().extendPointerWithSExt())
4565 return RValue::get(Builder.CreateSExt(Result, Int64Ty,
"extend.sext"));
4567 return RValue::get(Builder.CreateZExt(Result, Int64Ty,
"extend.zext"));
4569 case Builtin::BI__builtin_setjmp: {
4574 Value *FrameAddr = Builder.CreateCall(
4575 CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy),
4576 ConstantInt::get(Int32Ty, 0));
4577 Builder.CreateStore(FrameAddr, Buf);
4580 Value *StackAddr = Builder.CreateStackSave();
4583 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
4584 Builder.CreateStore(StackAddr, StackSaveSlot);
4587 Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
4590 case Builtin::BI__builtin_longjmp: {
4594 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
4597 Builder.CreateUnreachable();
4600 EmitBlock(createBasicBlock(
"longjmp.cont"));
4604 case Builtin::BI__builtin_launder: {
4607 Value *Ptr = EmitScalarExpr(Arg);
4609 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
4613 case Builtin::BI__sync_fetch_and_add:
4614 case Builtin::BI__sync_fetch_and_sub:
4615 case Builtin::BI__sync_fetch_and_or:
4616 case Builtin::BI__sync_fetch_and_and:
4617 case Builtin::BI__sync_fetch_and_xor:
4618 case Builtin::BI__sync_fetch_and_nand:
4619 case Builtin::BI__sync_add_and_fetch:
4620 case Builtin::BI__sync_sub_and_fetch:
4621 case Builtin::BI__sync_and_and_fetch:
4622 case Builtin::BI__sync_or_and_fetch:
4623 case Builtin::BI__sync_xor_and_fetch:
4624 case Builtin::BI__sync_nand_and_fetch:
4625 case Builtin::BI__sync_val_compare_and_swap:
4626 case Builtin::BI__sync_bool_compare_and_swap:
4627 case Builtin::BI__sync_lock_test_and_set:
4628 case Builtin::BI__sync_lock_release:
4629 case Builtin::BI__sync_swap:
4630 llvm_unreachable(
"Shouldn't make it through sema");
4631 case Builtin::BI__sync_fetch_and_add_1:
4632 case Builtin::BI__sync_fetch_and_add_2:
4633 case Builtin::BI__sync_fetch_and_add_4:
4634 case Builtin::BI__sync_fetch_and_add_8:
4635 case Builtin::BI__sync_fetch_and_add_16:
4637 case Builtin::BI__sync_fetch_and_sub_1:
4638 case Builtin::BI__sync_fetch_and_sub_2:
4639 case Builtin::BI__sync_fetch_and_sub_4:
4640 case Builtin::BI__sync_fetch_and_sub_8:
4641 case Builtin::BI__sync_fetch_and_sub_16:
4643 case Builtin::BI__sync_fetch_and_or_1:
4644 case Builtin::BI__sync_fetch_and_or_2:
4645 case Builtin::BI__sync_fetch_and_or_4:
4646 case Builtin::BI__sync_fetch_and_or_8:
4647 case Builtin::BI__sync_fetch_and_or_16:
4649 case Builtin::BI__sync_fetch_and_and_1:
4650 case Builtin::BI__sync_fetch_and_and_2:
4651 case Builtin::BI__sync_fetch_and_and_4:
4652 case Builtin::BI__sync_fetch_and_and_8:
4653 case Builtin::BI__sync_fetch_and_and_16:
4655 case Builtin::BI__sync_fetch_and_xor_1:
4656 case Builtin::BI__sync_fetch_and_xor_2:
4657 case Builtin::BI__sync_fetch_and_xor_4:
4658 case Builtin::BI__sync_fetch_and_xor_8:
4659 case Builtin::BI__sync_fetch_and_xor_16:
4661 case Builtin::BI__sync_fetch_and_nand_1:
4662 case Builtin::BI__sync_fetch_and_nand_2:
4663 case Builtin::BI__sync_fetch_and_nand_4:
4664 case Builtin::BI__sync_fetch_and_nand_8:
4665 case Builtin::BI__sync_fetch_and_nand_16:
4669 case Builtin::BI__sync_fetch_and_min:
4671 case Builtin::BI__sync_fetch_and_max:
4673 case Builtin::BI__sync_fetch_and_umin:
4675 case Builtin::BI__sync_fetch_and_umax:
4678 case Builtin::BI__sync_add_and_fetch_1:
4679 case Builtin::BI__sync_add_and_fetch_2:
4680 case Builtin::BI__sync_add_and_fetch_4:
4681 case Builtin::BI__sync_add_and_fetch_8:
4682 case Builtin::BI__sync_add_and_fetch_16:
4685 case Builtin::BI__sync_sub_and_fetch_1:
4686 case Builtin::BI__sync_sub_and_fetch_2:
4687 case Builtin::BI__sync_sub_and_fetch_4:
4688 case Builtin::BI__sync_sub_and_fetch_8:
4689 case Builtin::BI__sync_sub_and_fetch_16:
4692 case Builtin::BI__sync_and_and_fetch_1:
4693 case Builtin::BI__sync_and_and_fetch_2:
4694 case Builtin::BI__sync_and_and_fetch_4:
4695 case Builtin::BI__sync_and_and_fetch_8:
4696 case Builtin::BI__sync_and_and_fetch_16:
4699 case Builtin::BI__sync_or_and_fetch_1:
4700 case Builtin::BI__sync_or_and_fetch_2:
4701 case Builtin::BI__sync_or_and_fetch_4:
4702 case Builtin::BI__sync_or_and_fetch_8:
4703 case Builtin::BI__sync_or_and_fetch_16:
4705 llvm::Instruction::Or);
4706 case Builtin::BI__sync_xor_and_fetch_1:
4707 case Builtin::BI__sync_xor_and_fetch_2:
4708 case Builtin::BI__sync_xor_and_fetch_4:
4709 case Builtin::BI__sync_xor_and_fetch_8:
4710 case Builtin::BI__sync_xor_and_fetch_16:
4712 llvm::Instruction::Xor);
4713 case Builtin::BI__sync_nand_and_fetch_1:
4714 case Builtin::BI__sync_nand_and_fetch_2:
4715 case Builtin::BI__sync_nand_and_fetch_4:
4716 case Builtin::BI__sync_nand_and_fetch_8:
4717 case Builtin::BI__sync_nand_and_fetch_16:
4721 case Builtin::BI__sync_val_compare_and_swap_1:
4722 case Builtin::BI__sync_val_compare_and_swap_2:
4723 case Builtin::BI__sync_val_compare_and_swap_4:
4724 case Builtin::BI__sync_val_compare_and_swap_8:
4725 case Builtin::BI__sync_val_compare_and_swap_16:
4728 case Builtin::BI__sync_bool_compare_and_swap_1:
4729 case Builtin::BI__sync_bool_compare_and_swap_2:
4730 case Builtin::BI__sync_bool_compare_and_swap_4:
4731 case Builtin::BI__sync_bool_compare_and_swap_8:
4732 case Builtin::BI__sync_bool_compare_and_swap_16:
4735 case Builtin::BI__sync_swap_1:
4736 case Builtin::BI__sync_swap_2:
4737 case Builtin::BI__sync_swap_4:
4738 case Builtin::BI__sync_swap_8:
4739 case Builtin::BI__sync_swap_16:
4742 case Builtin::BI__sync_lock_test_and_set_1:
4743 case Builtin::BI__sync_lock_test_and_set_2:
4744 case Builtin::BI__sync_lock_test_and_set_4:
4745 case Builtin::BI__sync_lock_test_and_set_8:
4746 case Builtin::BI__sync_lock_test_and_set_16:
4749 case Builtin::BI__sync_lock_release_1:
4750 case Builtin::BI__sync_lock_release_2:
4751 case Builtin::BI__sync_lock_release_4:
4752 case Builtin::BI__sync_lock_release_8:
4753 case Builtin::BI__sync_lock_release_16: {
4757 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
4758 getContext().getTypeSize(ElTy));
4759 llvm::StoreInst *
Store =
4760 Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
4761 Store->setAtomic(llvm::AtomicOrdering::Release);
4765 case Builtin::BI__sync_synchronize: {
4773 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4777 case Builtin::BI__builtin_nontemporal_load:
4779 case Builtin::BI__builtin_nontemporal_store:
4781 case Builtin::BI__c11_atomic_is_lock_free:
4782 case Builtin::BI__atomic_is_lock_free: {
4786 const char *LibCallName =
"__atomic_is_lock_free";
4789 getContext().getSizeType());
4790 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4792 getContext().VoidPtrTy);
4795 getContext().VoidPtrTy);
4797 CGM.getTypes().arrangeBuiltinFunctionCall(E->
getType(), Args);
4798 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
4799 llvm::FunctionCallee
Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
4804 case Builtin::BI__atomic_test_and_set: {
4814 Value *NewVal = Builder.getInt8(1);
4816 if (isa<llvm::ConstantInt>(Order)) {
4817 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4818 AtomicRMWInst *Result =
nullptr;
4822 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4823 llvm::AtomicOrdering::Monotonic);
4827 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4828 llvm::AtomicOrdering::Acquire);
4831 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4832 llvm::AtomicOrdering::Release);
4836 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4837 llvm::AtomicOrdering::AcquireRelease);
4840 Result = Builder.CreateAtomicRMW(
4841 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4842 llvm::AtomicOrdering::SequentiallyConsistent);
4845 Result->setVolatile(Volatile);
4846 return RValue::get(Builder.CreateIsNotNull(Result,
"tobool"));
4849 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
4851 llvm::BasicBlock *BBs[5] = {
4852 createBasicBlock(
"monotonic", CurFn),
4853 createBasicBlock(
"acquire", CurFn),
4854 createBasicBlock(
"release", CurFn),
4855 createBasicBlock(
"acqrel", CurFn),
4856 createBasicBlock(
"seqcst", CurFn)
4858 llvm::AtomicOrdering Orders[5] = {
4859 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
4860 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
4861 llvm::AtomicOrdering::SequentiallyConsistent};
4863 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4864 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
4866 Builder.SetInsertPoint(ContBB);
4867 PHINode *Result = Builder.CreatePHI(Int8Ty, 5,
"was_set");
4869 for (
unsigned i = 0; i < 5; ++i) {
4870 Builder.SetInsertPoint(BBs[i]);
4871 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
4872 Ptr, NewVal, Orders[i]);
4873 RMW->setVolatile(Volatile);
4874 Result->addIncoming(RMW, BBs[i]);
4875 Builder.CreateBr(ContBB);
4878 SI->addCase(Builder.getInt32(0), BBs[0]);
4879 SI->addCase(Builder.getInt32(1), BBs[1]);
4880 SI->addCase(Builder.getInt32(2), BBs[1]);
4881 SI->addCase(Builder.getInt32(3), BBs[2]);
4882 SI->addCase(Builder.getInt32(4), BBs[3]);
4883 SI->addCase(Builder.getInt32(5), BBs[4]);
4885 Builder.SetInsertPoint(ContBB);
4886 return RValue::get(Builder.CreateIsNotNull(Result,
"tobool"));
4889 case Builtin::BI__atomic_clear: {
4896 Value *NewVal = Builder.getInt8(0);
4898 if (isa<llvm::ConstantInt>(Order)) {
4899 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4900 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
4904 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4907 Store->setOrdering(llvm::AtomicOrdering::Release);
4910 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4916 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
4918 llvm::BasicBlock *BBs[3] = {
4919 createBasicBlock(
"monotonic", CurFn),
4920 createBasicBlock(
"release", CurFn),
4921 createBasicBlock(
"seqcst", CurFn)
4923 llvm::AtomicOrdering Orders[3] = {
4924 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
4925 llvm::AtomicOrdering::SequentiallyConsistent};
4927 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4928 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
4930 for (
unsigned i = 0; i < 3; ++i) {
4931 Builder.SetInsertPoint(BBs[i]);
4932 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
4933 Store->setOrdering(Orders[i]);
4934 Builder.CreateBr(ContBB);
4937 SI->addCase(Builder.getInt32(0), BBs[0]);
4938 SI->addCase(Builder.getInt32(3), BBs[1]);
4939 SI->addCase(Builder.getInt32(5), BBs[2]);
4941 Builder.SetInsertPoint(ContBB);
4945 case Builtin::BI__atomic_thread_fence:
4946 case Builtin::BI__atomic_signal_fence:
4947 case Builtin::BI__c11_atomic_thread_fence:
4948 case Builtin::BI__c11_atomic_signal_fence: {
4950 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4951 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4952 SSID = llvm::SyncScope::SingleThread;
4956 if (isa<llvm::ConstantInt>(Order)) {
4957 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4964 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4967 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4970 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4973 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4979 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
4980 AcquireBB = createBasicBlock(
"acquire", CurFn);
4981 ReleaseBB = createBasicBlock(
"release", CurFn);
4982 AcqRelBB = createBasicBlock(
"acqrel", CurFn);
4983 SeqCstBB = createBasicBlock(
"seqcst", CurFn);
4984 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
4986 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4987 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
4989 Builder.SetInsertPoint(AcquireBB);
4990 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4991 Builder.CreateBr(ContBB);
4992 SI->addCase(Builder.getInt32(1), AcquireBB);
4993 SI->addCase(Builder.getInt32(2), AcquireBB);
4995 Builder.SetInsertPoint(ReleaseBB);
4996 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4997 Builder.CreateBr(ContBB);
4998 SI->addCase(Builder.getInt32(3), ReleaseBB);
5000 Builder.SetInsertPoint(AcqRelBB);
5001 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
5002 Builder.CreateBr(ContBB);
5003 SI->addCase(Builder.getInt32(4), AcqRelBB);
5005 Builder.SetInsertPoint(SeqCstBB);
5006 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5007 Builder.CreateBr(ContBB);
5008 SI->addCase(Builder.getInt32(5), SeqCstBB);
5010 Builder.SetInsertPoint(ContBB);
5014 case Builtin::BI__builtin_signbit:
5015 case Builtin::BI__builtin_signbitf:
5016 case Builtin::BI__builtin_signbitl: {
5021 case Builtin::BI__warn_memset_zero_len:
5023 case Builtin::BI__annotation: {
5027 const auto *Str = cast<StringLiteral>(Arg->IgnoreParenCasts());
5028 assert(Str->getCharByteWidth() == 2);
5029 StringRef WideBytes = Str->getBytes();
5030 std::string StrUtf8;
5031 if (!convertUTF16ToUTF8String(
5032 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
5033 CGM.ErrorUnsupported(E,
"non-UTF16 __annotation argument");
5036 Strings.push_back(llvm::MDString::get(getLLVMContext(), StrUtf8));
5041 CGM.getIntrinsic(llvm::Intrinsic::codeview_annotation, {});
5042 MDTuple *StrTuple = MDTuple::get(getLLVMContext(), Strings);
5043 Builder.CreateCall(F, MetadataAsValue::get(getLLVMContext(), StrTuple));
5046 case Builtin::BI__builtin_annotation: {
5047 llvm::Value *AnnVal = EmitScalarExpr(E->
getArg(0));
5049 CGM.getIntrinsic(llvm::Intrinsic::annotation,
5050 {AnnVal->getType(), CGM.ConstGlobalsPtrTy});
5055 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
5057 EmitAnnotationCall(F, AnnVal, Str, E->
getExprLoc(),
nullptr));
5059 case Builtin::BI__builtin_addcb:
5060 case Builtin::BI__builtin_addcs:
5061 case Builtin::BI__builtin_addc:
5062 case Builtin::BI__builtin_addcl:
5063 case Builtin::BI__builtin_addcll:
5064 case Builtin::BI__builtin_subcb:
5065 case Builtin::BI__builtin_subcs:
5066 case Builtin::BI__builtin_subc:
5067 case Builtin::BI__builtin_subcl:
5068 case Builtin::BI__builtin_subcll: {
5088 llvm::Value *
X = EmitScalarExpr(E->
getArg(0));
5089 llvm::Value *Y = EmitScalarExpr(E->
getArg(1));
5090 llvm::Value *Carryin = EmitScalarExpr(E->
getArg(2));
5095 switch (BuiltinID) {
5096 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
5097 case Builtin::BI__builtin_addcb:
5098 case Builtin::BI__builtin_addcs:
5099 case Builtin::BI__builtin_addc:
5100 case Builtin::BI__builtin_addcl:
5101 case Builtin::BI__builtin_addcll:
5102 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5104 case Builtin::BI__builtin_subcb:
5105 case Builtin::BI__builtin_subcs:
5106 case Builtin::BI__builtin_subc:
5107 case Builtin::BI__builtin_subcl:
5108 case Builtin::BI__builtin_subcll:
5109 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5114 llvm::Value *Carry1;
5117 llvm::Value *Carry2;
5119 Sum1, Carryin, Carry2);
5120 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
5122 Builder.CreateStore(CarryOut, CarryOutPtr);
5126 case Builtin::BI__builtin_add_overflow:
5127 case Builtin::BI__builtin_sub_overflow:
5128 case Builtin::BI__builtin_mul_overflow: {
5136 WidthAndSignedness LeftInfo =
5138 WidthAndSignedness RightInfo =
5140 WidthAndSignedness ResultInfo =
5147 RightInfo, ResultArg, ResultQTy,
5153 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
5156 WidthAndSignedness EncompassingInfo =
5159 llvm::Type *EncompassingLLVMTy =
5160 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
5162 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
5165 switch (BuiltinID) {
5167 llvm_unreachable(
"Unknown overflow builtin id.");
5168 case Builtin::BI__builtin_add_overflow:
5169 IntrinsicId = EncompassingInfo.Signed
5170 ? llvm::Intrinsic::sadd_with_overflow
5171 : llvm::Intrinsic::uadd_with_overflow;
5173 case Builtin::BI__builtin_sub_overflow:
5174 IntrinsicId = EncompassingInfo.Signed
5175 ? llvm::Intrinsic::ssub_with_overflow
5176 : llvm::Intrinsic::usub_with_overflow;
5178 case Builtin::BI__builtin_mul_overflow:
5179 IntrinsicId = EncompassingInfo.Signed
5180 ? llvm::Intrinsic::smul_with_overflow
5181 : llvm::Intrinsic::umul_with_overflow;
5185 llvm::Value *Left = EmitScalarExpr(LeftArg);
5186 llvm::Value *Right = EmitScalarExpr(RightArg);
5190 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
5191 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
5194 llvm::Value *Overflow, *Result;
5197 if (EncompassingInfo.Width > ResultInfo.Width) {
5200 llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
5204 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
5205 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
5206 llvm::Value *TruncationOverflow =
5207 Builder.CreateICmpNE(Result, ResultTruncExt);
5209 Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
5210 Result = ResultTrunc;
5216 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
5221 case Builtin::BI__builtin_uadd_overflow:
5222 case Builtin::BI__builtin_uaddl_overflow:
5223 case Builtin::BI__builtin_uaddll_overflow:
5224 case Builtin::BI__builtin_usub_overflow:
5225 case Builtin::BI__builtin_usubl_overflow:
5226 case Builtin::BI__builtin_usubll_overflow:
5227 case Builtin::BI__builtin_umul_overflow:
5228 case Builtin::BI__builtin_umull_overflow:
5229 case Builtin::BI__builtin_umulll_overflow:
5230 case Builtin::BI__builtin_sadd_overflow:
5231 case Builtin::BI__builtin_saddl_overflow:
5232 case Builtin::BI__builtin_saddll_overflow:
5233 case Builtin::BI__builtin_ssub_overflow:
5234 case Builtin::BI__builtin_ssubl_overflow:
5235 case Builtin::BI__builtin_ssubll_overflow:
5236 case Builtin::BI__builtin_smul_overflow:
5237 case Builtin::BI__builtin_smull_overflow:
5238 case Builtin::BI__builtin_smulll_overflow: {
5243 llvm::Value *
X = EmitScalarExpr(E->
getArg(0));
5244 llvm::Value *Y = EmitScalarExpr(E->
getArg(1));
5249 switch (BuiltinID) {
5250 default: llvm_unreachable(
"Unknown overflow builtin id.");
5251 case Builtin::BI__builtin_uadd_overflow:
5252 case Builtin::BI__builtin_uaddl_overflow:
5253 case Builtin::BI__builtin_uaddll_overflow:
5254 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5256 case Builtin::BI__builtin_usub_overflow:
5257 case Builtin::BI__builtin_usubl_overflow:
5258 case Builtin::BI__builtin_usubll_overflow:
5259 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5261 case Builtin::BI__builtin_umul_overflow:
5262 case Builtin::BI__builtin_umull_overflow:
5263 case Builtin::BI__builtin_umulll_overflow:
5264 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5266 case Builtin::BI__builtin_sadd_overflow:
5267 case Builtin::BI__builtin_saddl_overflow:
5268 case Builtin::BI__builtin_saddll_overflow:
5269 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5271 case Builtin::BI__builtin_ssub_overflow:
5272 case Builtin::BI__builtin_ssubl_overflow:
5273 case Builtin::BI__builtin_ssubll_overflow:
5274 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5276 case Builtin::BI__builtin_smul_overflow:
5277 case Builtin::BI__builtin_smull_overflow:
5278 case Builtin::BI__builtin_smulll_overflow:
5279 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5286 Builder.CreateStore(Sum, SumOutPtr);
5290 case Builtin::BIaddressof:
5291 case Builtin::BI__addressof:
5292 case Builtin::BI__builtin_addressof:
5294 case Builtin::BI__builtin_function_start:
5297 case Builtin::BI__builtin_operator_new:
5298 return EmitBuiltinNewDeleteCall(
5300 case Builtin::BI__builtin_operator_delete:
5301 EmitBuiltinNewDeleteCall(
5305 case Builtin::BI__builtin_is_aligned:
5306 return EmitBuiltinIsAligned(E);
5307 case Builtin::BI__builtin_align_up:
5308 return EmitBuiltinAlignTo(E,
true);
5309 case Builtin::BI__builtin_align_down:
5310 return EmitBuiltinAlignTo(E,
false);
5312 case Builtin::BI__noop:
5315 case Builtin::BI__builtin_call_with_static_chain: {
5318 return EmitCall(Call->getCallee()->getType(),
5319 EmitCallee(Call->getCallee()), Call, ReturnValue,
5320 EmitScalarExpr(Chain));
5322 case Builtin::BI_InterlockedExchange8:
5323 case Builtin::BI_InterlockedExchange16:
5324 case Builtin::BI_InterlockedExchange:
5325 case Builtin::BI_InterlockedExchangePointer:
5327 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
5328 case Builtin::BI_InterlockedCompareExchangePointer:
5329 case Builtin::BI_InterlockedCompareExchangePointer_nf: {
5331 llvm::IntegerType *IntType = IntegerType::get(
5332 getLLVMContext(), getContext().getTypeSize(E->
getType()));
5336 llvm::Value *Exchange = EmitScalarExpr(E->
getArg(1));
5337 RTy = Exchange->getType();
5338 Exchange = Builder.CreatePtrToInt(Exchange, IntType);
5340 llvm::Value *Comparand =
5341 Builder.CreatePtrToInt(EmitScalarExpr(E->
getArg(2)), IntType);
5344 BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
5345 AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
5347 auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
5348 Ordering, Ordering);
5349 Result->setVolatile(
true);
5351 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
5355 case Builtin::BI_InterlockedCompareExchange8:
5356 case Builtin::BI_InterlockedCompareExchange16:
5357 case Builtin::BI_InterlockedCompareExchange:
5358 case Builtin::BI_InterlockedCompareExchange64:
5360 case Builtin::BI_InterlockedIncrement16:
5361 case Builtin::BI_InterlockedIncrement:
5363 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
5364 case Builtin::BI_InterlockedDecrement16:
5365 case Builtin::BI_InterlockedDecrement:
5367 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
5368 case Builtin::BI_InterlockedAnd8:
5369 case Builtin::BI_InterlockedAnd16:
5370 case Builtin::BI_InterlockedAnd:
5371 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
5372 case Builtin::BI_InterlockedExchangeAdd8:
5373 case Builtin::BI_InterlockedExchangeAdd16:
5374 case Builtin::BI_InterlockedExchangeAdd:
5376 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
5377 case Builtin::BI_InterlockedExchangeSub8:
5378 case Builtin::BI_InterlockedExchangeSub16:
5379 case Builtin::BI_InterlockedExchangeSub:
5381 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
5382 case Builtin::BI_InterlockedOr8:
5383 case Builtin::BI_InterlockedOr16:
5384 case Builtin::BI_InterlockedOr:
5385 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
5386 case Builtin::BI_InterlockedXor8:
5387 case Builtin::BI_InterlockedXor16:
5388 case Builtin::BI_InterlockedXor:
5389 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
5391 case Builtin::BI_bittest64:
5392 case Builtin::BI_bittest:
5393 case Builtin::BI_bittestandcomplement64:
5394 case Builtin::BI_bittestandcomplement:
5395 case Builtin::BI_bittestandreset64:
5396 case Builtin::BI_bittestandreset:
5397 case Builtin::BI_bittestandset64:
5398 case Builtin::BI_bittestandset:
5399 case Builtin::BI_interlockedbittestandreset:
5400 case Builtin::BI_interlockedbittestandreset64:
5401 case Builtin::BI_interlockedbittestandset64:
5402 case Builtin::BI_interlockedbittestandset:
5403 case Builtin::BI_interlockedbittestandset_acq:
5404 case Builtin::BI_interlockedbittestandset_rel:
5405 case Builtin::BI_interlockedbittestandset_nf:
5406 case Builtin::BI_interlockedbittestandreset_acq:
5407 case Builtin::BI_interlockedbittestandreset_rel:
5408 case Builtin::BI_interlockedbittestandreset_nf:
5413 case Builtin::BI__iso_volatile_load8:
5414 case Builtin::BI__iso_volatile_load16:
5415 case Builtin::BI__iso_volatile_load32:
5416 case Builtin::BI__iso_volatile_load64:
5418 case Builtin::BI__iso_volatile_store8:
5419 case Builtin::BI__iso_volatile_store16:
5420 case Builtin::BI__iso_volatile_store32:
5421 case Builtin::BI__iso_volatile_store64:
5424 case Builtin::BI__builtin_ptrauth_auth:
5425 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5426 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5427 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5428 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5429 case Builtin::BI__builtin_ptrauth_strip: {
5433 Args.push_back(EmitScalarExpr(argExpr));
5436 llvm::Type *OrigValueType = Args[0]->getType();
5437 if (OrigValueType->isPointerTy())
5438 Args[0] = Builder.CreatePtrToInt(Args[0], IntPtrTy);
5440 switch (BuiltinID) {
5441 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5442 if (Args[4]->getType()->isPointerTy())
5443 Args[4] = Builder.CreatePtrToInt(Args[4], IntPtrTy);
5446 case Builtin::BI__builtin_ptrauth_auth:
5447 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5448 if (Args[2]->getType()->isPointerTy())
5449 Args[2] = Builder.CreatePtrToInt(Args[2], IntPtrTy);
5452 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5453 if (Args[1]->getType()->isPointerTy())
5454 Args[1] = Builder.CreatePtrToInt(Args[1], IntPtrTy);
5457 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5458 case Builtin::BI__builtin_ptrauth_strip:
5463 auto IntrinsicID = [&]() ->
unsigned {
5464 switch (BuiltinID) {
5465 case Builtin::BI__builtin_ptrauth_auth:
5466 return llvm::Intrinsic::ptrauth_auth;
5467 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5468 return llvm::Intrinsic::ptrauth_resign;
5469 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5470 return llvm::Intrinsic::ptrauth_blend;
5471 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5472 return llvm::Intrinsic::ptrauth_sign_generic;
5473 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5474 return llvm::Intrinsic::ptrauth_sign;
5475 case Builtin::BI__builtin_ptrauth_strip:
5478 llvm_unreachable(
"bad ptrauth intrinsic");
5480 auto Intrinsic = CGM.getIntrinsic(IntrinsicID);
5481 llvm::Value *Result = EmitRuntimeCall(Intrinsic, Args);
5483 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5484 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5485 OrigValueType->isPointerTy()) {
5486 Result = Builder.CreateIntToPtr(Result, OrigValueType);
5491 case Builtin::BI__exception_code:
5492 case Builtin::BI_exception_code:
5494 case Builtin::BI__exception_info:
5495 case Builtin::BI_exception_info:
5497 case Builtin::BI__abnormal_termination:
5498 case Builtin::BI_abnormal_termination:
5500 case Builtin::BI_setjmpex:
5501 if (getTarget().getTriple().isOSMSVCRT() && E->
getNumArgs() == 1 &&
5505 case Builtin::BI_setjmp:
5506 if (getTarget().getTriple().isOSMSVCRT() && E->
getNumArgs() == 1 &&
5508 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
5510 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5517 case Builtin::BImove:
5518 case Builtin::BImove_if_noexcept:
5519 case Builtin::BIforward:
5520 case Builtin::BIforward_like:
5521 case Builtin::BIas_const:
5523 case Builtin::BI__GetExceptionInfo: {
5524 if (llvm::GlobalVariable *GV =
5530 case Builtin::BI__fastfail:
5531 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::__fastfail, E));
5533 case Builtin::BI__builtin_coro_id:
5534 return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
5535 case Builtin::BI__builtin_coro_promise:
5536 return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
5537 case Builtin::BI__builtin_coro_resume:
5538 EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
5540 case Builtin::BI__builtin_coro_frame:
5541 return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
5542 case Builtin::BI__builtin_coro_noop:
5543 return EmitCoroutineIntrinsic(E, Intrinsic::coro_noop);
5544 case Builtin::BI__builtin_coro_free:
5545 return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
5546 case Builtin::BI__builtin_coro_destroy:
5547 EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
5549 case Builtin::BI__builtin_coro_done:
5550 return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
5551 case Builtin::BI__builtin_coro_alloc:
5552 return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
5553 case Builtin::BI__builtin_coro_begin:
5554 return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
5555 case Builtin::BI__builtin_coro_end:
5556 return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
5557 case Builtin::BI__builtin_coro_suspend:
5558 return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
5559 case Builtin::BI__builtin_coro_size:
5560 return EmitCoroutineIntrinsic(E, Intrinsic::coro_size);
5561 case Builtin::BI__builtin_coro_align:
5562 return EmitCoroutineIntrinsic(E, Intrinsic::coro_align);
5565 case Builtin::BIread_pipe:
5566 case Builtin::BIwrite_pipe: {
5568 *Arg1 = EmitScalarExpr(E->
getArg(1));
5574 unsigned GenericAS =
5576 llvm::Type *I8PTy = llvm::PointerType::get(getLLVMContext(), GenericAS);
5580 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
5584 llvm::Type *ArgTys[] = {Arg0->
getType(), I8PTy, Int32Ty, Int32Ty};
5585 llvm::FunctionType *FTy = llvm::FunctionType::get(
5587 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
5589 EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
5590 {Arg0, BCast, PacketSize, PacketAlign}));
5593 "Illegal number of parameters to pipe function");
5594 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
5597 llvm::Type *ArgTys[] = {Arg0->
getType(), Arg1->getType(), Int32Ty, I8PTy,
5600 *Arg3 = EmitScalarExpr(E->
getArg(3));
5601 llvm::FunctionType *FTy = llvm::FunctionType::get(
5603 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
5606 if (Arg2->
getType() != Int32Ty)
5607 Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
5609 EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
5610 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
5615 case Builtin::BIreserve_read_pipe:
5616 case Builtin::BIreserve_write_pipe:
5617 case Builtin::BIwork_group_reserve_read_pipe:
5618 case Builtin::BIwork_group_reserve_write_pipe:
5619 case Builtin::BIsub_group_reserve_read_pipe:
5620 case Builtin::BIsub_group_reserve_write_pipe: {
5623 if (BuiltinID == Builtin::BIreserve_read_pipe)
5624 Name =
"__reserve_read_pipe";
5625 else if (BuiltinID == Builtin::BIreserve_write_pipe)
5626 Name =
"__reserve_write_pipe";
5627 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
5628 Name =
"__work_group_reserve_read_pipe";
5629 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
5630 Name =
"__work_group_reserve_write_pipe";
5631 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
5632 Name =
"__sub_group_reserve_read_pipe";
5634 Name =
"__sub_group_reserve_write_pipe";
5637 *Arg1 = EmitScalarExpr(E->
getArg(1));
5638 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
5644 llvm::Type *ArgTys[] = {Arg0->
getType(), Int32Ty, Int32Ty, Int32Ty};
5645 llvm::FunctionType *FTy = llvm::FunctionType::get(
5649 if (Arg1->getType() != Int32Ty)
5650 Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
5651 return RValue::get(EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
5652 {Arg0, Arg1, PacketSize, PacketAlign}));
5656 case Builtin::BIcommit_read_pipe:
5657 case Builtin::BIcommit_write_pipe:
5658 case Builtin::BIwork_group_commit_read_pipe:
5659 case Builtin::BIwork_group_commit_write_pipe:
5660 case Builtin::BIsub_group_commit_read_pipe:
5661 case Builtin::BIsub_group_commit_write_pipe: {
5663 if (BuiltinID == Builtin::BIcommit_read_pipe)
5664 Name =
"__commit_read_pipe";
5665 else if (BuiltinID == Builtin::BIcommit_write_pipe)
5666 Name =
"__commit_write_pipe";
5667 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
5668 Name =
"__work_group_commit_read_pipe";
5669 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
5670 Name =
"__work_group_commit_write_pipe";
5671 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
5672 Name =
"__sub_group_commit_read_pipe";
5674 Name =
"__sub_group_commit_write_pipe";
5677 *Arg1 = EmitScalarExpr(E->
getArg(1));
5683 llvm::Type *ArgTys[] = {Arg0->
getType(), Arg1->getType(), Int32Ty, Int32Ty};
5684 llvm::FunctionType *FTy =
5685 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
5688 return RValue::get(EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
5689 {Arg0, Arg1, PacketSize, PacketAlign}));
5692 case Builtin::BIget_pipe_num_packets:
5693 case Builtin::BIget_pipe_max_packets: {
5694 const char *BaseName;
5696 if (BuiltinID == Builtin::BIget_pipe_num_packets)
5697 BaseName =
"__get_pipe_num_packets";
5699 BaseName =
"__get_pipe_max_packets";
5700 std::string Name = std::string(BaseName) +
5701 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
5708 llvm::Type *ArgTys[] = {Arg0->
getType(), Int32Ty, Int32Ty};
5709 llvm::FunctionType *FTy = llvm::FunctionType::get(
5712 return RValue::get(EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
5713 {Arg0, PacketSize, PacketAlign}));
5717 case Builtin::BIto_global:
5718 case Builtin::BIto_local:
5719 case Builtin::BIto_private: {
5720 auto Arg0 = EmitScalarExpr(E->
getArg(0));
5721 auto NewArgT = llvm::PointerType::get(
5724 auto NewRetT = llvm::PointerType::get(
5726 CGM.getContext().getTargetAddressSpace(
5728 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
5729 llvm::Value *NewArg;
5730 if (Arg0->getType()->getPointerAddressSpace() !=
5731 NewArgT->getPointerAddressSpace())
5732 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
5734 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
5737 EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
5738 return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
5750 case Builtin::BIenqueue_kernel: {
5754 llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
5755 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5758 llvm::Value *Queue = EmitScalarExpr(E->
getArg(0));
5759 llvm::Value *Flags = EmitScalarExpr(E->
getArg(1));
5767 Name =
"__enqueue_kernel_basic";
5768 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, GenericVoidPtrTy,
5770 llvm::FunctionType *FTy = llvm::FunctionType::get(
5774 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(3));
5776 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5777 llvm::Value *
Block =
5778 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5780 AttrBuilder B(Builder.getContext());
5782 llvm::AttributeList ByValAttrSet =
5783 llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
5786 EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
5787 {Queue, Flags, Range, Kernel, Block});
5788 RTCall->setAttributes(ByValAttrSet);
5791 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
5795 auto CreateArrayForSizeVar = [=](
unsigned First)
5796 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5798 QualType SizeArrayTy = getContext().getConstantArrayType(
5799 getContext().getSizeType(), ArraySize,
nullptr,
5802 auto Tmp = CreateMemTemp(SizeArrayTy,
"block_sizes");
5803 llvm::Value *TmpPtr = Tmp.getPointer();
5804 llvm::Value *TmpSize = EmitLifetimeStart(
5805 CGM.getDataLayout().getTypeAllocSize(Tmp.getElementType()), TmpPtr);
5806 llvm::Value *ElemPtr;
5809 auto *Zero = llvm::ConstantInt::get(IntTy, 0);
5810 for (
unsigned I =
First; I < NumArgs; ++I) {
5811 auto *Index = llvm::ConstantInt::get(IntTy, I -
First);
5812 auto *GEP = Builder.CreateGEP(Tmp.getElementType(), TmpPtr,
5817 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->
getArg(I)), SizeTy);
5818 Builder.CreateAlignedStore(
5819 V, GEP, CGM.getDataLayout().getPrefTypeAlign(SizeTy));
5821 return std::tie(ElemPtr, TmpSize, TmpPtr);
5827 Name =
"__enqueue_kernel_varargs";
5829 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(3));
5831 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5832 auto *
Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5833 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5834 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
5838 llvm::Value *
const Args[] = {Queue, Flags,
5840 Block, ConstantInt::get(IntTy, NumArgs - 4),
5842 llvm::Type *
const ArgTys[] = {
5843 QueueTy, IntTy, RangeTy, GenericVoidPtrTy,
5844 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
5846 llvm::FunctionType *FTy = llvm::FunctionType::get(Int32Ty, ArgTys,
false);
5848 EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Args));
5850 EmitLifetimeEnd(TmpSize, TmpPtr);
5855 llvm::PointerType *PtrTy = llvm::PointerType::get(
5856 CGM.getLLVMContext(),
5859 llvm::Value *NumEvents =
5860 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->
getArg(3)), Int32Ty);
5865 llvm::Value *EventWaitList =
nullptr;
5868 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
5872 ? EmitArrayToPointerDecay(E->
getArg(4)).emitRawPointer(*
this)
5873 : EmitScalarExpr(E->
getArg(4));
5875 EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy);
5877 llvm::Value *EventRet =
nullptr;
5880 EventRet = llvm::ConstantPointerNull::get(PtrTy);
5883 Builder.CreatePointerCast(EmitScalarExpr(E->
getArg(5)), PtrTy);
5887 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(6));
5889 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5890 llvm::Value *
Block =
5891 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5893 std::vector<llvm::Type *> ArgTys = {
5894 QueueTy, Int32Ty, RangeTy, Int32Ty,
5895 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
5897 std::vector<llvm::Value *> Args = {Queue, Flags, Range,
5898 NumEvents, EventWaitList, EventRet,
5903 Name =
"__enqueue_kernel_basic_events";
5904 llvm::FunctionType *FTy = llvm::FunctionType::get(
5907 EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
5912 Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
5913 ArgTys.push_back(Int32Ty);
5914 Name =
"__enqueue_kernel_events_varargs";
5916 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5917 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
5918 Args.push_back(ElemPtr);
5919 ArgTys.push_back(ElemPtr->getType());
5921 llvm::FunctionType *FTy = llvm::FunctionType::get(
5924 RValue::get(EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
5927 EmitLifetimeEnd(TmpSize, TmpPtr);
5930 llvm_unreachable(
"Unexpected enqueue_kernel signature");
5934 case Builtin::BIget_kernel_work_group_size: {
5935 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5938 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(0));
5940 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5941 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5943 CGM.CreateRuntimeFunction(
5944 llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5946 "__get_kernel_work_group_size_impl"),
5949 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
5950 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5953 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(0));
5955 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5956 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5958 CGM.CreateRuntimeFunction(
5959 llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5961 "__get_kernel_preferred_work_group_size_multiple_impl"),
5964 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
5965 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
5966 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5971 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(1));
5973 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5974 Value *
Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5976 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
5977 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
5978 :
"__get_kernel_sub_group_count_for_ndrange_impl";
5980 CGM.CreateRuntimeFunction(
5981 llvm::FunctionType::get(
5982 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
5985 {NDRange, Kernel, Block}));
5987 case Builtin::BI__builtin_store_half:
5988 case Builtin::BI__builtin_store_halff: {
5991 Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
5992 Builder.CreateStore(HalfVal,
Address);
5995 case Builtin::BI__builtin_load_half: {
5998 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
6000 case Builtin::BI__builtin_load_halff: {
6003 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
6005 case Builtin::BI__builtin_printf:
6006 case Builtin::BIprintf:
6007 if (getTarget().getTriple().isNVPTX() ||
6008 getTarget().getTriple().isAMDGCN()) {
6009 if (getLangOpts().OpenMPIsTargetDevice)
6010 return EmitOpenMPDevicePrintfCallExpr(E);
6011 if (getTarget().getTriple().isNVPTX())
6012 return EmitNVPTXDevicePrintfCallExpr(E);
6013 if (getTarget().getTriple().isAMDGCN() && getLangOpts().
HIP)
6014 return EmitAMDGPUDevicePrintfCallExpr(E);
6018 case Builtin::BI__builtin_canonicalize:
6019 case Builtin::BI__builtin_canonicalizef:
6020 case Builtin::BI__builtin_canonicalizef16:
6021 case Builtin::BI__builtin_canonicalizel:
6024 case Builtin::BI__builtin_thread_pointer: {
6025 if (!getContext().getTargetInfo().isTLSSupported())
6026 CGM.ErrorUnsupported(E,
"__builtin_thread_pointer");
6030 case Builtin::BI__builtin_os_log_format:
6031 return emitBuiltinOSLogFormat(*E);
6033 case Builtin::BI__xray_customevent: {
6034 if (!ShouldXRayInstrumentFunction())
6037 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6041 if (
const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6042 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
6045 Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
6046 auto FTy = F->getFunctionType();
6047 auto Arg0 = E->
getArg(0);
6048 auto Arg0Val = EmitScalarExpr(Arg0);
6049 auto Arg0Ty = Arg0->getType();
6050 auto PTy0 = FTy->getParamType(0);
6051 if (PTy0 != Arg0Val->getType()) {
6052 if (Arg0Ty->isArrayType())
6053 Arg0Val = EmitArrayToPointerDecay(Arg0).emitRawPointer(*
this);
6055 Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
6057 auto Arg1 = EmitScalarExpr(E->
getArg(1));
6058 auto PTy1 = FTy->getParamType(1);
6059 if (PTy1 != Arg1->getType())
6060 Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
6061 return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
6064 case Builtin::BI__xray_typedevent: {
6068 if (!ShouldXRayInstrumentFunction())
6071 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6075 if (
const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6076 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
6079 Function *F = CGM.getIntrinsic(Intrinsic::xray_typedevent);
6080 auto FTy = F->getFunctionType();
6081 auto Arg0 = EmitScalarExpr(E->
getArg(0));
6082 auto PTy0 = FTy->getParamType(0);
6083 if (PTy0 != Arg0->getType())
6084 Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
6085 auto Arg1 = E->
getArg(1);
6086 auto Arg1Val = EmitScalarExpr(Arg1);
6087 auto Arg1Ty = Arg1->getType();
6088 auto PTy1 = FTy->getParamType(1);
6089 if (PTy1 != Arg1Val->getType()) {
6090 if (Arg1Ty->isArrayType())
6091 Arg1Val = EmitArrayToPointerDecay(Arg1).emitRawPointer(*
this);
6093 Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
6095 auto Arg2 = EmitScalarExpr(E->
getArg(2));
6096 auto PTy2 = FTy->getParamType(2);
6097 if (PTy2 != Arg2->getType())
6098 Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
6099 return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
6102 case Builtin::BI__builtin_ms_va_start:
6103 case Builtin::BI__builtin_ms_va_end:
6105 EmitVAStartEnd(EmitMSVAListRef(E->
getArg(0)).emitRawPointer(*
this),
6106 BuiltinID == Builtin::BI__builtin_ms_va_start));
6108 case Builtin::BI__builtin_ms_va_copy: {
6121 Value *ArgPtr = Builder.CreateLoad(SrcAddr,
"ap.val");
6122 return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
6126 case Builtin::BI__builtin_intel_fpga_reg:
6127 return EmitIntelFPGARegBuiltin(E, ReturnValue);
6128 case Builtin::BI__builtin_intel_fpga_mem:
6129 return EmitIntelFPGAMemBuiltin(E);
6130 case Builtin::BI__builtin_intel_sycl_ptr_annotation:
6131 return EmitIntelSYCLPtrAnnotationBuiltin(E);
6132 case Builtin::BI__builtin_intel_sycl_alloca:
6133 case Builtin::BI__builtin_intel_sycl_alloca_with_align:
6134 return EmitIntelSYCLAllocaBuiltin(BuiltinID, E, ReturnValue);
6135 case Builtin::BI__builtin_get_device_side_mangled_name: {
6136 auto Name = CGM.getCUDARuntime().getDeviceSideName(
6138 auto Str = CGM.GetAddrOfConstantCString(Name,
"");
6139 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0),
6140 llvm::ConstantInt::get(SizeTy, 0)};
6141 auto *Ptr = llvm::ConstantExpr::getGetElementPtr(Str.getElementType(),
6142 Str.getPointer(), Zeros);
6150 if (getContext().
BuiltinInfo.isLibFunction(BuiltinID))
6152 CGM.getBuiltinLibFunction(FD, BuiltinID));
6156 if (getContext().
BuiltinInfo.isPredefinedLibFunction(BuiltinID))
6158 *
this, FD, E, cast<llvm::Constant>(EmitScalarExpr(E->
getCallee())));
6165 checkTargetFeatures(E, FD);
6167 if (
unsigned VectorWidth = getContext().
BuiltinInfo.getRequiredVectorWidth(BuiltinID))
6168 LargestVectorWidth =
std::max(LargestVectorWidth, VectorWidth);
6171 StringRef Name = getContext().BuiltinInfo.getName(BuiltinID);
6174 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
6175 if (!Prefix.empty()) {
6176 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
6180 if (IntrinsicID == Intrinsic::not_intrinsic)
6181 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
6184 if (IntrinsicID != Intrinsic::not_intrinsic) {
6189 unsigned ICEArguments = 0;
6191 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
6194 Function *F = CGM.getIntrinsic(IntrinsicID);
6195 llvm::FunctionType *FTy = F->getFunctionType();
6197 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
6198 Value *ArgValue = EmitScalarOrConstFoldImmArg(ICEArguments, i, E);
6201 llvm::Type *PTy = FTy->getParamType(i);
6202 if (PTy != ArgValue->
getType()) {
6204 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
6205 if (PtrTy->getAddressSpace() !=
6206 ArgValue->
getType()->getPointerAddressSpace()) {
6207 ArgValue = Builder.CreateAddrSpaceCast(
6208 ArgValue, llvm::PointerType::get(getLLVMContext(),
6209 PtrTy->getAddressSpace()));
6215 if (PTy->isX86_AMXTy())
6216 ArgValue = Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6217 {ArgValue->
getType()}, {ArgValue});
6219 ArgValue = Builder.CreateBitCast(ArgValue, PTy);
6222 Args.push_back(ArgValue);
6225 Value *
V = Builder.CreateCall(F, Args);
6228 llvm::Type *RetTy = VoidTy;
6230 RetTy = ConvertType(BuiltinRetType);
6232 if (RetTy !=
V->getType()) {
6234 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
6235 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
6236 V = Builder.CreateAddrSpaceCast(
6237 V, llvm::PointerType::get(getLLVMContext(),
6238 PtrTy->getAddressSpace()));
6244 if (
V->getType()->isX86_AMXTy())
6245 V = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6248 V = Builder.CreateBitCast(
V, RetTy);
6251 if (RetTy->isVoidTy())
6268 if (
Value *
V = EmitTargetBuiltinExpr(BuiltinID, E, ReturnValue)) {
6271 if (
V->getType()->isVoidTy())
6276 ReturnValue.isVolatile());
6278 llvm_unreachable(
"No current target builtin returns complex");
6280 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6284 if (
Value *
V = EmitHLSLBuiltinExpr(BuiltinID, E))
6287 if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice)
6290 ErrorUnsupported(E,
"builtin function");
6293 return GetUndefRValue(E->
getType());
6297 unsigned BuiltinID,
const CallExpr *E,
6299 llvm::Triple::ArchType Arch) {
6311 case llvm::Triple::arm:
6312 case llvm::Triple::armeb:
6313 case llvm::Triple::thumb:
6314 case llvm::Triple::thumbeb:
6316 case llvm::Triple::aarch64:
6317 case llvm::Triple::aarch64_32:
6318 case llvm::Triple::aarch64_be:
6320 case llvm::Triple::bpfeb:
6321 case llvm::Triple::bpfel:
6323 case llvm::Triple::x86:
6324 case llvm::Triple::x86_64:
6326 case llvm::Triple::ppc:
6327 case llvm::Triple::ppcle:
6328 case llvm::Triple::ppc64:
6329 case llvm::Triple::ppc64le:
6331 case llvm::Triple::r600:
6332 case llvm::Triple::amdgcn:
6334 case llvm::Triple::systemz:
6336 case llvm::Triple::nvptx:
6337 case llvm::Triple::nvptx64:
6339 case llvm::Triple::wasm32:
6340 case llvm::Triple::wasm64:
6342 case llvm::Triple::hexagon:
6344 case llvm::Triple::riscv32:
6345 case llvm::Triple::riscv64:
6355 if (getContext().
BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
6356 assert(getContext().getAuxTargetInfo() &&
"Missing aux target info");
6358 this, getContext().
BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
6359 ReturnValue, getContext().getAuxTargetInfo()->getTriple().getArch());
6363 getTarget().getTriple().getArch());
6368 bool HasLegalHalfType =
true,
6370 bool AllowBFloatArgsAndRet =
true) {
6371 int IsQuad = TypeFlags.
isQuad();
6375 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6378 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6380 if (AllowBFloatArgsAndRet)
6381 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6383 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6385 if (HasLegalHalfType)
6386 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
6388 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6390 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6393 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6398 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
6400 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
6402 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6404 llvm_unreachable(
"Unknown vector element type!");
6409 int IsQuad = IntTypeFlags.
isQuad();
6412 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
6414 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
6416 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
6418 llvm_unreachable(
"Type can't be converted to floating-point!");
6423 const ElementCount &Count) {
6424 Value *SV = llvm::ConstantVector::getSplat(Count, C);
6425 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
6429 ElementCount EC = cast<llvm::VectorType>(
V->getType())->getElementCount();
6430 return EmitNeonSplat(
V, C, EC);
6435 unsigned shift,
bool rightshift) {
6437 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6438 ai != ae; ++ai, ++j) {
6439 if (F->isConstrainedFPIntrinsic())
6440 if (ai->getType()->isMetadataTy())
6442 if (shift > 0 && shift == j)
6443 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
6445 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(),
name);
6448 if (F->isConstrainedFPIntrinsic())
6449 return Builder.CreateConstrainedFPCall(F, Ops,
name);
6451 return Builder.CreateCall(F, Ops,
name);
6456 int SV = cast<ConstantInt>(
V)->getSExtValue();
6457 return ConstantInt::get(Ty, neg ? -SV : SV);
6462 llvm::Type *Ty,
bool usgn,
6464 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6466 int ShiftAmt = cast<ConstantInt>(
Shift)->getSExtValue();
6467 int EltSize = VTy->getScalarSizeInBits();
6469 Vec = Builder.CreateBitCast(Vec, Ty);
6473 if (ShiftAmt == EltSize) {
6476 return llvm::ConstantAggregateZero::get(VTy);
6481 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
6485 Shift = EmitNeonShiftVector(
Shift, Ty,
false);
6487 return Builder.CreateLShr(Vec,
Shift,
name);
6489 return Builder.CreateAShr(Vec,
Shift,
name);
6515 struct ARMVectorIntrinsicInfo {
6516 const char *NameHint;
6518 unsigned LLVMIntrinsic;
6519 unsigned AltLLVMIntrinsic;
6522 bool operator<(
unsigned RHSBuiltinID)
const {
6523 return BuiltinID < RHSBuiltinID;
6525 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
6526 return BuiltinID < TE.BuiltinID;
6531 #define NEONMAP0(NameBase) \
6532 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
6534 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6535 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6536 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6538 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
6539 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6540 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
6544 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
6551 NEONMAP1(vabs_v, arm_neon_vabs, 0),
6552 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
6556 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
6557 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
6558 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
6559 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
6560 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
6561 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
6562 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
6563 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
6564 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
6577 NEONMAP1(vcage_v, arm_neon_vacge, 0),
6578 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
6579 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
6580 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
6581 NEONMAP1(vcale_v, arm_neon_vacge, 0),
6582 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
6583 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
6584 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
6601 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
6604 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
6606 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6607 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6608 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6609 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6610 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6611 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6612 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6613 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6614 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6621 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
6622 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
6623 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
6624 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
6625 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
6626 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
6627 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
6628 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
6629 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
6630 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
6631 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
6632 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
6633 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
6634 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
6635 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
6636 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
6637 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
6638 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
6639 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
6640 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
6641 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
6642 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
6643 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
6644 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
6645 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
6646 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
6647 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
6648 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
6649 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
6650 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
6651 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
6652 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
6653 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
6654 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
6655 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
6656 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
6657 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
6658 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
6659 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
6660 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
6661 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
6662 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
6663 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
6664 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
6665 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
6666 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
6667 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
6668 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
6669 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
6673 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6674 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6675 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6676 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6677 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6678 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6679 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6680 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6681 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6688 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
6689 NEONMAP1(vdot_u32, arm_neon_udot, 0),
6690 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
6691 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
6701 NEONMAP1(vld1_v, arm_neon_vld1, 0),
6702 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
6703 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
6704 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
6706 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
6707 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
6708 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
6709 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
6710 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
6711 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
6712 NEONMAP1(vld2_v, arm_neon_vld2, 0),
6713 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
6714 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
6715 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
6716 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
6717 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
6718 NEONMAP1(vld3_v, arm_neon_vld3, 0),
6719 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
6720 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
6721 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
6722 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
6723 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
6724 NEONMAP1(vld4_v, arm_neon_vld4, 0),
6725 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
6726 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
6727 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
6736 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
6737 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
6755 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
6756 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
6780 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
6781 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
6785 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6786 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6809 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6810 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6814 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
6815 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
6816 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
6817 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
6818 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
6819 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
6828 NEONMAP1(vst1_v, arm_neon_vst1, 0),
6829 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
6830 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
6831 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
6832 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
6833 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
6834 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
6835 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
6836 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
6837 NEONMAP1(vst2_v, arm_neon_vst2, 0),
6838 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
6839 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
6840 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
6841 NEONMAP1(vst3_v, arm_neon_vst3, 0),
6842 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
6843 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
6844 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
6845 NEONMAP1(vst4_v, arm_neon_vst4, 0),
6846 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
6847 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
6853 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
6854 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
6855 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
6863 NEONMAP1(__a64_vcvtq_low_bf16_f32, aarch64_neon_bfcvtn, 0),
6868 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
6869 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
6874 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
6875 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
6876 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
6877 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
6886 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
6887 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
6888 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
6889 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
6890 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
6901 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
6902 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
6903 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
6904 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
6905 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
6906 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
6907 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
6908 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
6945 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
6948 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
6950 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6951 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6952 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6953 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6954 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6955 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6956 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6957 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6958 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6959 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6963 NEONMAP1(vcvtq_high_bf16_f32, aarch64_neon_bfcvtn2, 0),
6964 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6965 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6966 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6967 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6968 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6969 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6970 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6971 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6972 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6973 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6975 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
6976 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
6977 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
6978 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
6991 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
6992 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
6993 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
6994 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
6995 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
6996 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
6997 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
6998 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
7003 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
7004 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
7005 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
7006 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
7007 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
7008 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
7009 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
7010 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
7023 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
7024 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
7025 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
7026 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7028 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
7029 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7044 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7045 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7047 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7048 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7056 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
7057 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
7061 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
7062 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7063 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7090 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7091 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7095 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
7096 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
7097 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
7098 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
7099 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
7100 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
7101 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
7102 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
7103 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
7104 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
7113 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
7114 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
7115 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
7116 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
7117 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
7118 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
7119 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
7120 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
7121 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
7122 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
7123 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
7124 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
7125 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
7126 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
7127 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
7131 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
7132 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
7133 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
7134 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
7172 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
7191 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
7212 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
7240 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
7321 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
7322 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
7323 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
7324 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
7378 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
7379 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
7380 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
7381 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
7382 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
7383 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
7384 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
7385 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
7386 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
7387 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
7388 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
7389 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
7390 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
7391 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
7392 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
7393 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
7394 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
7395 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
7396 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
7397 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
7398 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
7399 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
7400 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
7401 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
7402 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
7403 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
7404 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
7405 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
7406 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
7407 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
7408 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
7409 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
7410 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
7411 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
7412 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
7413 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
7414 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
7415 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
7416 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
7417 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
7418 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
7419 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
7420 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
7421 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
7422 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
7423 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
7424 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
7425 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
7426 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
7427 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
7428 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
7429 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
7430 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
7431 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
7432 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
7433 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
7434 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
7435 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
7436 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
7437 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
7438 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
7439 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
7440 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
7441 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
7442 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
7443 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
7444 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
7445 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
7446 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
7447 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
7448 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
7449 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
7450 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
7451 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
7452 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
7453 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
7454 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
7455 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
7456 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
7457 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
7458 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
7459 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
7460 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
7461 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
7462 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
7463 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
7464 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
7465 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
7466 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
7467 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
7468 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
7469 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
7470 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
7471 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
7472 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
7473 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
7474 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
7475 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
7476 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
7477 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
7478 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
7479 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
7480 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
7481 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
7482 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
7483 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
7484 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
7485 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
7486 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
7487 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
7488 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
7489 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
7490 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
7491 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
7492 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
7493 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
7494 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
7495 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
7496 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
7497 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
7498 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
7499 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
7500 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
7501 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
7502 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
7503 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
7504 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
7505 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
7509 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7510 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7511 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7512 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7513 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7514 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7515 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7516 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7517 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7518 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7519 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7520 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7527 #define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7529 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7533 #define SVEMAP2(NameBase, TypeModifier) \
7534 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
7536 #define GET_SVE_LLVM_INTRINSIC_MAP
7537 #include "clang/Basic/arm_sve_builtin_cg.inc"
7538 #include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
7539 #undef GET_SVE_LLVM_INTRINSIC_MAP
7545 #define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7547 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7551 #define SMEMAP2(NameBase, TypeModifier) \
7552 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
7554 #define GET_SME_LLVM_INTRINSIC_MAP
7555 #include "clang/Basic/arm_sme_builtin_cg.inc"
7556 #undef GET_SME_LLVM_INTRINSIC_MAP
7569 static const ARMVectorIntrinsicInfo *
7571 unsigned BuiltinID,
bool &MapProvenSorted) {
7574 if (!MapProvenSorted) {
7575 assert(llvm::is_sorted(IntrinsicMap));
7576 MapProvenSorted =
true;
7580 const ARMVectorIntrinsicInfo *Builtin =
7581 llvm::lower_bound(IntrinsicMap, BuiltinID);
7583 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
7591 llvm::Type *ArgType,
7604 Ty = llvm::FixedVectorType::get(
7605 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
7612 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
7613 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
7617 Tys.push_back(ArgType);
7620 Tys.push_back(ArgType);
7623 Tys.push_back(FloatTy);
7625 return CGM.getIntrinsic(IntrinsicID, Tys);
7631 unsigned BuiltinID = SISDInfo.BuiltinID;
7632 unsigned int Int = SISDInfo.LLVMIntrinsic;
7633 unsigned Modifier = SISDInfo.TypeModifier;
7634 const char *
s = SISDInfo.NameHint;
7636 switch (BuiltinID) {
7637 case NEON::BI__builtin_neon_vcled_s64:
7638 case NEON::BI__builtin_neon_vcled_u64:
7639 case NEON::BI__builtin_neon_vcles_f32:
7640 case NEON::BI__builtin_neon_vcled_f64:
7641 case NEON::BI__builtin_neon_vcltd_s64:
7642 case NEON::BI__builtin_neon_vcltd_u64:
7643 case NEON::BI__builtin_neon_vclts_f32:
7644 case NEON::BI__builtin_neon_vcltd_f64:
7645 case NEON::BI__builtin_neon_vcales_f32:
7646 case NEON::BI__builtin_neon_vcaled_f64:
7647 case NEON::BI__builtin_neon_vcalts_f32:
7648 case NEON::BI__builtin_neon_vcaltd_f64:
7652 std::swap(Ops[0], Ops[1]);
7656 assert(Int &&
"Generic code assumes a valid intrinsic");
7664 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
7665 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
7666 ai != ae; ++ai, ++j) {
7667 llvm::Type *ArgTy = ai->getType();
7668 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
7669 ArgTy->getPrimitiveSizeInBits())
7672 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
7675 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
7676 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
7678 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
7683 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
7684 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
7685 return CGF.
Builder.CreateExtractElement(Result, C0);
7687 return CGF.
Builder.CreateBitCast(Result, ResultType,
s);
7691 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
7692 const char *NameHint,
unsigned Modifier,
const CallExpr *E,
7694 llvm::Triple::ArchType Arch) {
7697 std::optional<llvm::APSInt> NeonTypeConst =
7704 bool Usgn =
Type.isUnsigned();
7705 bool Quad =
Type.isQuad();
7706 const bool HasLegalHalfType = getTarget().hasLegalHalfType();
7707 const bool AllowBFloatArgsAndRet =
7708 getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
7710 llvm::FixedVectorType *VTy =
7711 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
7712 llvm::Type *Ty = VTy;
7716 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
7717 return Builder.getInt32(addr.getAlignment().getQuantity());
7720 unsigned Int = LLVMIntrinsic;
7722 Int = AltLLVMIntrinsic;
7724 switch (BuiltinID) {
7726 case NEON::BI__builtin_neon_splat_lane_v:
7727 case NEON::BI__builtin_neon_splat_laneq_v:
7728 case NEON::BI__builtin_neon_splatq_lane_v:
7729 case NEON::BI__builtin_neon_splatq_laneq_v: {
7730 auto NumElements = VTy->getElementCount();
7731 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
7732 NumElements = NumElements * 2;
7733 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
7734 NumElements = NumElements.divideCoefficientBy(2);
7736 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
7737 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
7739 case NEON::BI__builtin_neon_vpadd_v:
7740 case NEON::BI__builtin_neon_vpaddq_v:
7742 if (VTy->getElementType()->isFloatingPointTy() &&
7743 Int == Intrinsic::aarch64_neon_addp)
7744 Int = Intrinsic::aarch64_neon_faddp;
7746 case NEON::BI__builtin_neon_vabs_v:
7747 case NEON::BI__builtin_neon_vabsq_v:
7748 if (VTy->getElementType()->isFloatingPointTy())
7749 return EmitNeonCall(CGM.getIntrinsic(
Intrinsic::fabs, Ty), Ops,
"vabs");
7750 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops,
"vabs");
7751 case NEON::BI__builtin_neon_vadd_v:
7752 case NEON::BI__builtin_neon_vaddq_v: {
7753 llvm::Type *VTy = llvm::FixedVectorType::get(Int8Ty, Quad ? 16 : 8);
7754 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
7755 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
7756 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
7757 return Builder.CreateBitCast(Ops[0], Ty);
7759 case NEON::BI__builtin_neon_vaddhn_v: {
7760 llvm::FixedVectorType *SrcTy =
7761 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7764 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7765 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
7766 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
7769 Constant *ShiftAmt =
7770 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7771 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
7774 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
7776 case NEON::BI__builtin_neon_vcale_v:
7777 case NEON::BI__builtin_neon_vcaleq_v:
7778 case NEON::BI__builtin_neon_vcalt_v:
7779 case NEON::BI__builtin_neon_vcaltq_v:
7780 std::swap(Ops[0], Ops[1]);
7782 case NEON::BI__builtin_neon_vcage_v:
7783 case NEON::BI__builtin_neon_vcageq_v:
7784 case NEON::BI__builtin_neon_vcagt_v:
7785 case NEON::BI__builtin_neon_vcagtq_v: {
7787 switch (VTy->getScalarSizeInBits()) {
7788 default: llvm_unreachable(
"unexpected type");
7799 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
7800 llvm::Type *Tys[] = { VTy, VecFlt };
7801 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
7802 return EmitNeonCall(F, Ops, NameHint);
7804 case NEON::BI__builtin_neon_vceqz_v:
7805 case NEON::BI__builtin_neon_vceqzq_v:
7806 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
7807 ICmpInst::ICMP_EQ,
"vceqz");
7808 case NEON::BI__builtin_neon_vcgez_v:
7809 case NEON::BI__builtin_neon_vcgezq_v:
7810 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
7811 ICmpInst::ICMP_SGE,
"vcgez");
7812 case NEON::BI__builtin_neon_vclez_v:
7813 case NEON::BI__builtin_neon_vclezq_v:
7814 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
7815 ICmpInst::ICMP_SLE,
"vclez");
7816 case NEON::BI__builtin_neon_vcgtz_v:
7817 case NEON::BI__builtin_neon_vcgtzq_v:
7818 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
7819 ICmpInst::ICMP_SGT,
"vcgtz");
7820 case NEON::BI__builtin_neon_vcltz_v:
7821 case NEON::BI__builtin_neon_vcltzq_v:
7822 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
7823 ICmpInst::ICMP_SLT,
"vcltz");
7824 case NEON::BI__builtin_neon_vclz_v:
7825 case NEON::BI__builtin_neon_vclzq_v:
7828 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
7830 case NEON::BI__builtin_neon_vcvt_f32_v:
7831 case NEON::BI__builtin_neon_vcvtq_f32_v:
7832 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7835 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7836 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7837 case NEON::BI__builtin_neon_vcvt_f16_s16:
7838 case NEON::BI__builtin_neon_vcvt_f16_u16:
7839 case NEON::BI__builtin_neon_vcvtq_f16_s16:
7840 case NEON::BI__builtin_neon_vcvtq_f16_u16:
7841 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7844 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7845 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7846 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
7847 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
7848 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
7849 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
7851 Function *F = CGM.getIntrinsic(Int, Tys);
7852 return EmitNeonCall(F, Ops,
"vcvt_n");
7854 case NEON::BI__builtin_neon_vcvt_n_f32_v:
7855 case NEON::BI__builtin_neon_vcvt_n_f64_v:
7856 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
7857 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
7859 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
7860 Function *F = CGM.getIntrinsic(Int, Tys);
7861 return EmitNeonCall(F, Ops,
"vcvt_n");
7863 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
7864 case NEON::BI__builtin_neon_vcvt_n_s32_v:
7865 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
7866 case NEON::BI__builtin_neon_vcvt_n_u32_v:
7867 case NEON::BI__builtin_neon_vcvt_n_s64_v:
7868 case NEON::BI__builtin_neon_vcvt_n_u64_v:
7869 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
7870 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
7871 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
7872 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
7873 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
7874 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
7876 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
7877 return EmitNeonCall(F, Ops,
"vcvt_n");
7879 case NEON::BI__builtin_neon_vcvt_s32_v:
7880 case NEON::BI__builtin_neon_vcvt_u32_v:
7881 case NEON::BI__builtin_neon_vcvt_s64_v:
7882 case NEON::BI__builtin_neon_vcvt_u64_v:
7883 case NEON::BI__builtin_neon_vcvt_s16_f16:
7884 case NEON::BI__builtin_neon_vcvt_u16_f16:
7885 case NEON::BI__builtin_neon_vcvtq_s32_v:
7886 case NEON::BI__builtin_neon_vcvtq_u32_v:
7887 case NEON::BI__builtin_neon_vcvtq_s64_v:
7888 case NEON::BI__builtin_neon_vcvtq_u64_v:
7889 case NEON::BI__builtin_neon_vcvtq_s16_f16:
7890 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
7892 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
7893 : Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
7895 case NEON::BI__builtin_neon_vcvta_s16_f16:
7896 case NEON::BI__builtin_neon_vcvta_s32_v:
7897 case NEON::BI__builtin_neon_vcvta_s64_v:
7898 case NEON::BI__builtin_neon_vcvta_u16_f16:
7899 case NEON::BI__builtin_neon_vcvta_u32_v:
7900 case NEON::BI__builtin_neon_vcvta_u64_v:
7901 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
7902 case NEON::BI__builtin_neon_vcvtaq_s32_v:
7903 case NEON::BI__builtin_neon_vcvtaq_s64_v:
7904 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
7905 case NEON::BI__builtin_neon_vcvtaq_u32_v:
7906 case NEON::BI__builtin_neon_vcvtaq_u64_v:
7907 case NEON::BI__builtin_neon_vcvtn_s16_f16:
7908 case NEON::BI__builtin_neon_vcvtn_s32_v:
7909 case NEON::BI__builtin_neon_vcvtn_s64_v:
7910 case NEON::BI__builtin_neon_vcvtn_u16_f16:
7911 case NEON::BI__builtin_neon_vcvtn_u32_v:
7912 case NEON::BI__builtin_neon_vcvtn_u64_v:
7913 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
7914 case NEON::BI__builtin_neon_vcvtnq_s32_v:
7915 case NEON::BI__builtin_neon_vcvtnq_s64_v:
7916 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
7917 case NEON::BI__builtin_neon_vcvtnq_u32_v:
7918 case NEON::BI__builtin_neon_vcvtnq_u64_v:
7919 case NEON::BI__builtin_neon_vcvtp_s16_f16:
7920 case NEON::BI__builtin_neon_vcvtp_s32_v:
7921 case NEON::BI__builtin_neon_vcvtp_s64_v:
7922 case NEON::BI__builtin_neon_vcvtp_u16_f16:
7923 case NEON::BI__builtin_neon_vcvtp_u32_v:
7924 case NEON::BI__builtin_neon_vcvtp_u64_v:
7925 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
7926 case NEON::BI__builtin_neon_vcvtpq_s32_v:
7927 case NEON::BI__builtin_neon_vcvtpq_s64_v:
7928 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
7929 case NEON::BI__builtin_neon_vcvtpq_u32_v:
7930 case NEON::BI__builtin_neon_vcvtpq_u64_v:
7931 case NEON::BI__builtin_neon_vcvtm_s16_f16:
7932 case NEON::BI__builtin_neon_vcvtm_s32_v:
7933 case NEON::BI__builtin_neon_vcvtm_s64_v:
7934 case NEON::BI__builtin_neon_vcvtm_u16_f16:
7935 case NEON::BI__builtin_neon_vcvtm_u32_v:
7936 case NEON::BI__builtin_neon_vcvtm_u64_v:
7937 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
7938 case NEON::BI__builtin_neon_vcvtmq_s32_v:
7939 case NEON::BI__builtin_neon_vcvtmq_s64_v:
7940 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
7941 case NEON::BI__builtin_neon_vcvtmq_u32_v:
7942 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7944 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
7946 case NEON::BI__builtin_neon_vcvtx_f32_v: {
7947 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
7948 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
7951 case NEON::BI__builtin_neon_vext_v:
7952 case NEON::BI__builtin_neon_vextq_v: {
7953 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
7955 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7956 Indices.push_back(i+CV);
7958 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7959 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7960 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
7962 case NEON::BI__builtin_neon_vfma_v:
7963 case NEON::BI__builtin_neon_vfmaq_v: {
7964 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7965 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7966 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7970 *
this,
Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
7971 {Ops[1], Ops[2], Ops[0]});
7973 case NEON::BI__builtin_neon_vld1_v:
7974 case NEON::BI__builtin_neon_vld1q_v: {
7975 llvm::Type *Tys[] = {Ty, Int8PtrTy};
7976 Ops.push_back(getAlignmentValue32(PtrOp0));
7977 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"vld1");
7979 case NEON::BI__builtin_neon_vld1_x2_v:
7980 case NEON::BI__builtin_neon_vld1q_x2_v:
7981 case NEON::BI__builtin_neon_vld1_x3_v:
7982 case NEON::BI__builtin_neon_vld1q_x3_v:
7983 case NEON::BI__builtin_neon_vld1_x4_v:
7984 case NEON::BI__builtin_neon_vld1q_x4_v: {
7985 llvm::Type *Tys[2] = {VTy, UnqualPtrTy};
7986 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
7987 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld1xN");
7988 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7990 case NEON::BI__builtin_neon_vld2_v:
7991 case NEON::BI__builtin_neon_vld2q_v:
7992 case NEON::BI__builtin_neon_vld3_v:
7993 case NEON::BI__builtin_neon_vld3q_v:
7994 case NEON::BI__builtin_neon_vld4_v:
7995 case NEON::BI__builtin_neon_vld4q_v:
7996 case NEON::BI__builtin_neon_vld2_dup_v:
7997 case NEON::BI__builtin_neon_vld2q_dup_v:
7998 case NEON::BI__builtin_neon_vld3_dup_v:
7999 case NEON::BI__builtin_neon_vld3q_dup_v:
8000 case NEON::BI__builtin_neon_vld4_dup_v:
8001 case NEON::BI__builtin_neon_vld4q_dup_v: {
8002 llvm::Type *Tys[] = {Ty, Int8PtrTy};
8003 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
8004 Value *Align = getAlignmentValue32(PtrOp1);
8005 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
8006 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8008 case NEON::BI__builtin_neon_vld1_dup_v:
8009 case NEON::BI__builtin_neon_vld1q_dup_v: {
8010 Value *
V = PoisonValue::get(Ty);
8012 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
8013 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
8014 Ops[0] = Builder.CreateInsertElement(
V, Ld, CI);
8015 return EmitNeonSplat(Ops[0], CI);
8017 case NEON::BI__builtin_neon_vld2_lane_v:
8018 case NEON::BI__builtin_neon_vld2q_lane_v:
8019 case NEON::BI__builtin_neon_vld3_lane_v:
8020 case NEON::BI__builtin_neon_vld3q_lane_v:
8021 case NEON::BI__builtin_neon_vld4_lane_v:
8022 case NEON::BI__builtin_neon_vld4q_lane_v: {
8023 llvm::Type *Tys[] = {Ty, Int8PtrTy};
8024 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
8025 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
8026 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
8027 Ops.push_back(getAlignmentValue32(PtrOp1));
8028 Ops[1] = Builder.CreateCall(F,
ArrayRef(Ops).slice(1), NameHint);
8029 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8031 case NEON::BI__builtin_neon_vmovl_v: {
8032 llvm::FixedVectorType *DTy =
8033 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8034 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
8036 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
8037 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
8039 case NEON::BI__builtin_neon_vmovn_v: {
8040 llvm::FixedVectorType *QTy =
8041 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8042 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
8043 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
8045 case NEON::BI__builtin_neon_vmull_v:
8051 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
8052 Int =
Type.isPoly() ? (
unsigned)Intrinsic::arm_neon_vmullp : Int;
8053 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmull");
8054 case NEON::BI__builtin_neon_vpadal_v:
8055 case NEON::BI__builtin_neon_vpadalq_v: {
8057 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8059 llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
8061 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8062 llvm::Type *Tys[2] = { Ty, NarrowTy };
8063 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
8065 case NEON::BI__builtin_neon_vpaddl_v:
8066 case NEON::BI__builtin_neon_vpaddlq_v: {
8068 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8069 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
8071 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8072 llvm::Type *Tys[2] = { Ty, NarrowTy };
8073 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vpaddl");
8075 case NEON::BI__builtin_neon_vqdmlal_v:
8076 case NEON::BI__builtin_neon_vqdmlsl_v: {
8079 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps,
"vqdmlal");
8081 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
8083 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
8084 case NEON::BI__builtin_neon_vqdmulh_lane_v:
8085 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
8086 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
8087 auto *RTy = cast<llvm::FixedVectorType>(Ty);
8088 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
8089 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
8090 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
8091 RTy->getNumElements() * 2);
8092 llvm::Type *Tys[2] = {
8095 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
8097 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
8098 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
8099 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
8100 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
8101 llvm::Type *Tys[2] = {
8104 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
8106 case NEON::BI__builtin_neon_vqshl_n_v:
8107 case NEON::BI__builtin_neon_vqshlq_n_v:
8108 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshl_n",
8110 case NEON::BI__builtin_neon_vqshlu_n_v:
8111 case NEON::BI__builtin_neon_vqshluq_n_v:
8112 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshlu_n",
8114 case NEON::BI__builtin_neon_vrecpe_v:
8115 case NEON::BI__builtin_neon_vrecpeq_v:
8116 case NEON::BI__builtin_neon_vrsqrte_v:
8117 case NEON::BI__builtin_neon_vrsqrteq_v:
8118 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
8119 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
8120 case NEON::BI__builtin_neon_vrndi_v:
8121 case NEON::BI__builtin_neon_vrndiq_v:
8122 Int = Builder.getIsFPConstrained()
8123 ? Intrinsic::experimental_constrained_nearbyint
8125 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
8126 case NEON::BI__builtin_neon_vrshr_n_v:
8127 case NEON::BI__builtin_neon_vrshrq_n_v:
8128 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrshr_n",
8130 case NEON::BI__builtin_neon_vsha512hq_u64:
8131 case NEON::BI__builtin_neon_vsha512h2q_u64:
8132 case NEON::BI__builtin_neon_vsha512su0q_u64:
8133 case NEON::BI__builtin_neon_vsha512su1q_u64: {
8134 Function *F = CGM.getIntrinsic(Int);
8135 return EmitNeonCall(F, Ops,
"");
8137 case NEON::BI__builtin_neon_vshl_n_v:
8138 case NEON::BI__builtin_neon_vshlq_n_v:
8139 Ops[1] = EmitNeonShiftVector(Ops[1], Ty,
false);
8140 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
8142 case NEON::BI__builtin_neon_vshll_n_v: {
8143 llvm::FixedVectorType *SrcTy =
8144 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8145 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
8147 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
8149 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
8150 Ops[1] = EmitNeonShiftVector(Ops[1], VTy,
false);
8151 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
8153 case NEON::BI__builtin_neon_vshrn_n_v: {
8154 llvm::FixedVectorType *SrcTy =
8155 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8156 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
8157 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy,
false);
8159 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
8161 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
8162 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
8164 case NEON::BI__builtin_neon_vshr_n_v:
8165 case NEON::BI__builtin_neon_vshrq_n_v:
8166 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn,
"vshr_n");
8167 case NEON::BI__builtin_neon_vst1_v:
8168 case NEON::BI__builtin_neon_vst1q_v:
8169 case NEON::BI__builtin_neon_vst2_v:
8170 case NEON::BI__builtin_neon_vst2q_v:
8171 case NEON::BI__builtin_neon_vst3_v:
8172 case NEON::BI__builtin_neon_vst3q_v:
8173 case NEON::BI__builtin_neon_vst4_v:
8174 case NEON::BI__builtin_neon_vst4q_v:
8175 case NEON::BI__builtin_neon_vst2_lane_v:
8176 case NEON::BI__builtin_neon_vst2q_lane_v:
8177 case NEON::BI__builtin_neon_vst3_lane_v:
8178 case NEON::BI__builtin_neon_vst3q_lane_v:
8179 case NEON::BI__builtin_neon_vst4_lane_v:
8180 case NEON::BI__builtin_neon_vst4q_lane_v: {
8181 llvm::Type *Tys[] = {Int8PtrTy, Ty};
8182 Ops.push_back(getAlignmentValue32(PtrOp0));
8183 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"");
8185 case NEON::BI__builtin_neon_vsm3partw1q_u32:
8186 case NEON::BI__builtin_neon_vsm3partw2q_u32:
8187 case NEON::BI__builtin_neon_vsm3ss1q_u32:
8188 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
8189 case NEON::BI__builtin_neon_vsm4eq_u32: {
8190 Function *F = CGM.getIntrinsic(Int);
8191 return EmitNeonCall(F, Ops,
"");
8193 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
8194 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
8195 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
8196 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
8197 Function *F = CGM.getIntrinsic(Int);
8198 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
8199 return EmitNeonCall(F, Ops,
"");
8201 case NEON::BI__builtin_neon_vst1_x2_v:
8202 case NEON::BI__builtin_neon_vst1q_x2_v:
8203 case NEON::BI__builtin_neon_vst1_x3_v:
8204 case NEON::BI__builtin_neon_vst1q_x3_v:
8205 case NEON::BI__builtin_neon_vst1_x4_v:
8206 case NEON::BI__builtin_neon_vst1q_x4_v: {
8209 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
8210 Arch == llvm::Triple::aarch64_32) {
8211 llvm::Type *Tys[2] = {VTy, UnqualPtrTy};
8212 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
8213 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"");
8215 llvm::Type *Tys[2] = {UnqualPtrTy, VTy};
8216 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"");
8218 case NEON::BI__builtin_neon_vsubhn_v: {
8219 llvm::FixedVectorType *SrcTy =
8220 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8223 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
8224 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
8225 Ops[0] = Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
8228 Constant *ShiftAmt =
8229 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8230 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
8233 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
8235 case NEON::BI__builtin_neon_vtrn_v:
8236 case NEON::BI__builtin_neon_vtrnq_v: {
8237 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8238 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8239 Value *SV =
nullptr;
8241 for (
unsigned vi = 0; vi != 2; ++vi) {
8243 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8244 Indices.push_back(i+vi);
8245 Indices.push_back(i+e+vi);
8247 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8248 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8249 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8253 case NEON::BI__builtin_neon_vtst_v:
8254 case NEON::BI__builtin_neon_vtstq_v: {
8255 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8256 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8257 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
8258 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8259 ConstantAggregateZero::get(Ty));
8260 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
8262 case NEON::BI__builtin_neon_vuzp_v:
8263 case NEON::BI__builtin_neon_vuzpq_v: {
8264 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8265 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8266 Value *SV =
nullptr;
8268 for (
unsigned vi = 0; vi != 2; ++vi) {
8270 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8271 Indices.push_back(2*i+vi);
8273 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8274 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8275 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8279 case NEON::BI__builtin_neon_vxarq_u64: {
8280 Function *F = CGM.getIntrinsic(Int);
8281 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
8282 return EmitNeonCall(F, Ops,
"");
8284 case NEON::BI__builtin_neon_vzip_v:
8285 case NEON::BI__builtin_neon_vzipq_v: {
8286 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8287 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8288 Value *SV =
nullptr;
8290 for (
unsigned vi = 0; vi != 2; ++vi) {
8292 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8293 Indices.push_back((i + vi*e) >> 1);
8294 Indices.push_back(((i + vi*e) >> 1)+e);
8296 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8297 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8298 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8302 case NEON::BI__builtin_neon_vdot_s32:
8303 case NEON::BI__builtin_neon_vdot_u32:
8304 case NEON::BI__builtin_neon_vdotq_s32:
8305 case NEON::BI__builtin_neon_vdotq_u32: {
8307 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8308 llvm::Type *Tys[2] = { Ty, InputTy };
8309 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vdot");
8311 case NEON::BI__builtin_neon_vfmlal_low_f16:
8312 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
8314 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8315 llvm::Type *Tys[2] = { Ty, InputTy };
8316 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vfmlal_low");
8318 case NEON::BI__builtin_neon_vfmlsl_low_f16:
8319 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
8321 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8322 llvm::Type *Tys[2] = { Ty, InputTy };
8323 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vfmlsl_low");
8325 case NEON::BI__builtin_neon_vfmlal_high_f16:
8326 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
8328 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8329 llvm::Type *Tys[2] = { Ty, InputTy };
8330 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vfmlal_high");
8332 case NEON::BI__builtin_neon_vfmlsl_high_f16:
8333 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
8335 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8336 llvm::Type *Tys[2] = { Ty, InputTy };
8337 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vfmlsl_high");
8339 case NEON::BI__builtin_neon_vmmlaq_s32:
8340 case NEON::BI__builtin_neon_vmmlaq_u32: {
8342 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8343 llvm::Type *Tys[2] = { Ty, InputTy };
8344 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"vmmla");
8346 case NEON::BI__builtin_neon_vusmmlaq_s32: {
8348 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8349 llvm::Type *Tys[2] = { Ty, InputTy };
8350 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vusmmla");
8352 case NEON::BI__builtin_neon_vusdot_s32:
8353 case NEON::BI__builtin_neon_vusdotq_s32: {
8355 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8356 llvm::Type *Tys[2] = { Ty, InputTy };
8357 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vusdot");
8359 case NEON::BI__builtin_neon_vbfdot_f32:
8360 case NEON::BI__builtin_neon_vbfdotq_f32: {
8361 llvm::Type *InputTy =
8362 llvm::FixedVectorType::get(BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8363 llvm::Type *Tys[2] = { Ty, InputTy };
8364 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vbfdot");
8366 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
8367 llvm::Type *Tys[1] = { Ty };
8368 Function *F = CGM.getIntrinsic(Int, Tys);
8369 return EmitNeonCall(F, Ops,
"vcvtfp2bf");
8374 assert(Int &&
"Expected valid intrinsic number");
8377 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
8379 Value *Result = EmitNeonCall(F, Ops, NameHint);
8380 llvm::Type *ResultType = ConvertType(E->
getType());
8383 return Builder.CreateBitCast(Result, ResultType, NameHint);
8387 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
8388 const CmpInst::Predicate Ip,
const Twine &Name) {
8389 llvm::Type *OTy = Op->
getType();
8395 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
8396 OTy = BI->getOperand(0)->getType();
8398 Op = Builder.CreateBitCast(Op, OTy);
8399 if (OTy->getScalarType()->isFloatingPointTy()) {
8400 if (Fp == CmpInst::FCMP_OEQ)
8401 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8403 Op = Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8405 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8407 return Builder.CreateSExt(Op, Ty, Name);
8412 llvm::Type *ResTy,
unsigned IntID,
8416 TblOps.push_back(ExtOp);
8420 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8421 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8422 Indices.push_back(2*i);
8423 Indices.push_back(2*i+1);
8426 int PairPos = 0,
End = Ops.size() - 1;
8427 while (PairPos <
End) {
8428 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8429 Ops[PairPos+1], Indices,
8436 if (PairPos ==
End) {
8437 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
8438 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8439 ZeroTbl, Indices, Name));
8443 TblOps.push_back(IndexOp);
8449 Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
8451 switch (BuiltinID) {
8454 case clang::ARM::BI__builtin_arm_nop:
8457 case clang::ARM::BI__builtin_arm_yield:
8458 case clang::ARM::BI__yield:
8461 case clang::ARM::BI__builtin_arm_wfe:
8462 case clang::ARM::BI__wfe:
8465 case clang::ARM::BI__builtin_arm_wfi:
8466 case clang::ARM::BI__wfi:
8469 case clang::ARM::BI__builtin_arm_sev:
8470 case clang::ARM::BI__sev:
8473 case clang::ARM::BI__builtin_arm_sevl:
8474 case clang::ARM::BI__sevl:
8479 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
8480 llvm::ConstantInt::get(Int32Ty,
Value));
8492 llvm::Type *RegisterType,
8493 llvm::Type *ValueType,
bool isExecHi) {
8498 llvm::Value *Call = Builder.CreateCall(F, {Builder.getInt1(
true)});
8501 Value *Rt2 = Builder.CreateLShr(Call, 32);
8502 Rt2 = Builder.CreateTrunc(Rt2, CGF.
Int32Ty);
8514 llvm::Type *RegisterType,
8515 llvm::Type *ValueType,
8517 StringRef SysReg =
"") {
8519 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
8520 RegisterType->isIntegerTy(128)) &&
8521 "Unsupported size for register.");
8527 if (SysReg.empty()) {
8529 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8532 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
8533 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8534 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8536 llvm::Type *Types[] = { RegisterType };
8538 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8539 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8540 &&
"Can't fit 64-bit value in 32-bit register");
8546 : llvm::Intrinsic::read_register,
8548 llvm::Value *Call = Builder.CreateCall(F, Metadata);
8552 return Builder.CreateTrunc(Call, ValueType);
8554 if (ValueType->isPointerTy())
8556 return Builder.CreateIntToPtr(Call, ValueType);
8561 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
8565 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
8566 return Builder.CreateCall(F, { Metadata, ArgValue });
8569 if (ValueType->isPointerTy()) {
8571 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
8572 return Builder.CreateCall(F, { Metadata, ArgValue });
8575 return Builder.CreateCall(F, { Metadata, ArgValue });
8581 switch (BuiltinID) {
8583 case NEON::BI__builtin_neon_vget_lane_i8:
8584 case NEON::BI__builtin_neon_vget_lane_i16:
8585 case NEON::BI__builtin_neon_vget_lane_bf16:
8586 case NEON::BI__builtin_neon_vget_lane_i32:
8587 case NEON::BI__builtin_neon_vget_lane_i64:
8588 case NEON::BI__builtin_neon_vget_lane_f32:
8589 case NEON::BI__builtin_neon_vgetq_lane_i8:
8590 case NEON::BI__builtin_neon_vgetq_lane_i16:
8591 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8592 case NEON::BI__builtin_neon_vgetq_lane_i32:
8593 case NEON::BI__builtin_neon_vgetq_lane_i64:
8594 case NEON::BI__builtin_neon_vgetq_lane_f32:
8595 case NEON::BI__builtin_neon_vduph_lane_bf16:
8596 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8597 case NEON::BI__builtin_neon_vset_lane_i8:
8598 case NEON::BI__builtin_neon_vset_lane_i16:
8599 case NEON::BI__builtin_neon_vset_lane_bf16:
8600 case NEON::BI__builtin_neon_vset_lane_i32:
8601 case NEON::BI__builtin_neon_vset_lane_i64:
8602 case NEON::BI__builtin_neon_vset_lane_f32:
8603 case NEON::BI__builtin_neon_vsetq_lane_i8:
8604 case NEON::BI__builtin_neon_vsetq_lane_i16:
8605 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8606 case NEON::BI__builtin_neon_vsetq_lane_i32:
8607 case NEON::BI__builtin_neon_vsetq_lane_i64:
8608 case NEON::BI__builtin_neon_vsetq_lane_f32:
8609 case NEON::BI__builtin_neon_vsha1h_u32:
8610 case NEON::BI__builtin_neon_vsha1cq_u32:
8611 case NEON::BI__builtin_neon_vsha1pq_u32:
8612 case NEON::BI__builtin_neon_vsha1mq_u32:
8613 case NEON::BI__builtin_neon_vcvth_bf16_f32:
8614 case clang::ARM::BI_MoveToCoprocessor:
8615 case clang::ARM::BI_MoveToCoprocessor2:
8624 llvm::Triple::ArchType Arch) {
8625 if (
auto Hint = GetValueForARMHint(BuiltinID))
8628 if (BuiltinID == clang::ARM::BI__emit) {
8629 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
8630 llvm::FunctionType *FTy =
8631 llvm::FunctionType::get(VoidTy,
false);
8635 llvm_unreachable(
"Sema will ensure that the parameter is constant");
8638 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
8640 llvm::InlineAsm *Emit =
8641 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
8643 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
8646 return Builder.CreateCall(Emit);
8649 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
8651 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
8654 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
8660 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
8663 return Builder.CreateCall(F, {
Address, RW, Locality, IsData});
8666 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
8667 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
8668 return Builder.CreateCall(
8669 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
8672 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
8673 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
8674 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
8675 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
8676 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(
false)});
8677 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
8678 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
8683 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
8684 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
8685 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls), Arg,
"cls");
8687 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
8688 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
8689 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
8693 if (BuiltinID == clang::ARM::BI__clear_cache) {
8694 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
8697 for (
unsigned i = 0; i < 2; i++)
8698 Ops[i] = EmitScalarExpr(E->
getArg(i));
8699 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->
getType());
8700 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
8701 StringRef Name = FD->
getName();
8702 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
8705 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
8706 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
8709 switch (BuiltinID) {
8710 default: llvm_unreachable(
"unexpected builtin");
8711 case clang::ARM::BI__builtin_arm_mcrr:
8712 F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
8714 case clang::ARM::BI__builtin_arm_mcrr2:
8715 F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
8731 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
8732 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
8733 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
8734 Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
8736 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
8739 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
8740 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
8743 switch (BuiltinID) {
8744 default: llvm_unreachable(
"unexpected builtin");
8745 case clang::ARM::BI__builtin_arm_mrrc:
8746 F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
8748 case clang::ARM::BI__builtin_arm_mrrc2:
8749 F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
8756 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
8761 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
8762 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
8763 Rt = Builder.CreateZExt(Rt, Int64Ty);
8764 Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
8766 Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
8767 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
8768 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
8770 return Builder.CreateBitCast(RtAndRt2, ConvertType(E->
getType()));
8773 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
8774 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8775 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
8776 getContext().getTypeSize(E->
getType()) == 64) ||
8777 BuiltinID == clang::ARM::BI__ldrexd) {
8780 switch (BuiltinID) {
8781 default: llvm_unreachable(
"unexpected builtin");
8782 case clang::ARM::BI__builtin_arm_ldaex:
8783 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
8785 case clang::ARM::BI__builtin_arm_ldrexd:
8786 case clang::ARM::BI__builtin_arm_ldrex:
8787 case clang::ARM::BI__ldrexd:
8788 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
8793 Value *Val = Builder.CreateCall(F, LdPtr,
"ldrexd");
8795 Value *Val0 = Builder.CreateExtractValue(Val, 1);
8796 Value *Val1 = Builder.CreateExtractValue(Val, 0);
8797 Val0 = Builder.CreateZExt(Val0, Int64Ty);
8798 Val1 = Builder.CreateZExt(Val1, Int64Ty);
8800 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
8801 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
8802 Val = Builder.CreateOr(Val, Val1);
8803 return Builder.CreateBitCast(Val, ConvertType(E->
getType()));
8806 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8807 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
8811 llvm::Type *RealResTy = ConvertType(Ty);
8813 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
8816 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
8817 : Intrinsic::arm_ldrex,
8819 CallInst *Val = Builder.CreateCall(F, LoadAddr,
"ldrex");
8821 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
8823 if (RealResTy->isPointerTy())
8824 return Builder.CreateIntToPtr(Val, RealResTy);
8826 llvm::Type *IntResTy = llvm::IntegerType::get(
8827 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
8828 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
8833 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
8834 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
8835 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
8838 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
8839 : Intrinsic::arm_strexd);
8840 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
8844 Builder.CreateStore(Val, Tmp);
8847 Val = Builder.CreateLoad(LdPtr);
8849 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
8850 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
8852 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
8855 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
8856 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
8862 llvm::Type *StoreTy =
8863 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
8865 if (StoreVal->
getType()->isPointerTy())
8866 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
8868 llvm::Type *IntTy = llvm::IntegerType::get(
8870 CGM.getDataLayout().getTypeSizeInBits(StoreVal->
getType()));
8871 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
8872 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
8876 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
8877 : Intrinsic::arm_strex,
8880 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
8882 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
8886 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
8887 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
8888 return Builder.CreateCall(F);
8893 switch (BuiltinID) {
8894 case clang::ARM::BI__builtin_arm_crc32b:
8895 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
8896 case clang::ARM::BI__builtin_arm_crc32cb:
8897 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
8898 case clang::ARM::BI__builtin_arm_crc32h:
8899 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
8900 case clang::ARM::BI__builtin_arm_crc32ch:
8901 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
8902 case clang::ARM::BI__builtin_arm_crc32w:
8903 case clang::ARM::BI__builtin_arm_crc32d:
8904 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
8905 case clang::ARM::BI__builtin_arm_crc32cw:
8906 case clang::ARM::BI__builtin_arm_crc32cd:
8907 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
8910 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
8916 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
8917 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
8918 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
8919 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
8920 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
8921 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
8923 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
8924 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
8925 return Builder.CreateCall(F, {Res, Arg1b});
8927 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
8929 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
8930 return Builder.CreateCall(F, {Arg0, Arg1});
8934 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8935 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8936 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8937 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
8938 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
8939 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
8942 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8943 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8944 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
8947 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8948 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
8950 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8951 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
8953 llvm::Type *ValueType;
8954 llvm::Type *RegisterType;
8955 if (IsPointerBuiltin) {
8956 ValueType = VoidPtrTy;
8957 RegisterType = Int32Ty;
8958 }
else if (Is64Bit) {
8959 ValueType = RegisterType = Int64Ty;
8961 ValueType = RegisterType = Int32Ty;
8968 if (BuiltinID == ARM::BI__builtin_sponentry) {
8969 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
8970 return Builder.CreateCall(F);
8976 return EmitMSVCBuiltinExpr(*MsvcIntId, E);
8979 if (
Value *Result = EmitARMMVEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
8982 if (
Value *Result = EmitARMCDEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
8987 return P.first == BuiltinID;
8990 BuiltinID = It->second;
8994 unsigned ICEArguments = 0;
8996 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
8999 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
9000 return Builder.getInt32(addr.getAlignment().getQuantity());
9007 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
9008 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
9010 switch (BuiltinID) {
9011 case NEON::BI__builtin_neon_vld1_v:
9012 case NEON::BI__builtin_neon_vld1q_v:
9013 case NEON::BI__builtin_neon_vld1q_lane_v:
9014 case NEON::BI__builtin_neon_vld1_lane_v:
9015 case NEON::BI__builtin_neon_vld1_dup_v:
9016 case NEON::BI__builtin_neon_vld1q_dup_v:
9017 case NEON::BI__builtin_neon_vst1_v:
9018 case NEON::BI__builtin_neon_vst1q_v:
9019 case NEON::BI__builtin_neon_vst1q_lane_v:
9020 case NEON::BI__builtin_neon_vst1_lane_v:
9021 case NEON::BI__builtin_neon_vst2_v:
9022 case NEON::BI__builtin_neon_vst2q_v:
9023 case NEON::BI__builtin_neon_vst2_lane_v:
9024 case NEON::BI__builtin_neon_vst2q_lane_v:
9025 case NEON::BI__builtin_neon_vst3_v:
9026 case NEON::BI__builtin_neon_vst3q_v:
9027 case NEON::BI__builtin_neon_vst3_lane_v:
9028 case NEON::BI__builtin_neon_vst3q_lane_v:
9029 case NEON::BI__builtin_neon_vst4_v:
9030 case NEON::BI__builtin_neon_vst4q_v:
9031 case NEON::BI__builtin_neon_vst4_lane_v:
9032 case NEON::BI__builtin_neon_vst4q_lane_v:
9041 switch (BuiltinID) {
9042 case NEON::BI__builtin_neon_vld2_v:
9043 case NEON::BI__builtin_neon_vld2q_v:
9044 case NEON::BI__builtin_neon_vld3_v:
9045 case NEON::BI__builtin_neon_vld3q_v:
9046 case NEON::BI__builtin_neon_vld4_v:
9047 case NEON::BI__builtin_neon_vld4q_v:
9048 case NEON::BI__builtin_neon_vld2_lane_v:
9049 case NEON::BI__builtin_neon_vld2q_lane_v:
9050 case NEON::BI__builtin_neon_vld3_lane_v:
9051 case NEON::BI__builtin_neon_vld3q_lane_v:
9052 case NEON::BI__builtin_neon_vld4_lane_v:
9053 case NEON::BI__builtin_neon_vld4q_lane_v:
9054 case NEON::BI__builtin_neon_vld2_dup_v:
9055 case NEON::BI__builtin_neon_vld2q_dup_v:
9056 case NEON::BI__builtin_neon_vld3_dup_v:
9057 case NEON::BI__builtin_neon_vld3q_dup_v:
9058 case NEON::BI__builtin_neon_vld4_dup_v:
9059 case NEON::BI__builtin_neon_vld4q_dup_v:
9068 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E));
9071 switch (BuiltinID) {
9074 case NEON::BI__builtin_neon_vget_lane_i8:
9075 case NEON::BI__builtin_neon_vget_lane_i16:
9076 case NEON::BI__builtin_neon_vget_lane_i32:
9077 case NEON::BI__builtin_neon_vget_lane_i64:
9078 case NEON::BI__builtin_neon_vget_lane_bf16:
9079 case NEON::BI__builtin_neon_vget_lane_f32:
9080 case NEON::BI__builtin_neon_vgetq_lane_i8:
9081 case NEON::BI__builtin_neon_vgetq_lane_i16:
9082 case NEON::BI__builtin_neon_vgetq_lane_i32:
9083 case NEON::BI__builtin_neon_vgetq_lane_i64:
9084 case NEON::BI__builtin_neon_vgetq_lane_bf16:
9085 case NEON::BI__builtin_neon_vgetq_lane_f32:
9086 case NEON::BI__builtin_neon_vduph_lane_bf16:
9087 case NEON::BI__builtin_neon_vduph_laneq_bf16:
9088 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
9090 case NEON::BI__builtin_neon_vrndns_f32: {
9092 llvm::Type *Tys[] = {Arg->
getType()};
9093 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vrintn, Tys);
9094 return Builder.CreateCall(F, {Arg},
"vrndn"); }
9096 case NEON::BI__builtin_neon_vset_lane_i8:
9097 case NEON::BI__builtin_neon_vset_lane_i16:
9098 case NEON::BI__builtin_neon_vset_lane_i32:
9099 case NEON::BI__builtin_neon_vset_lane_i64:
9100 case NEON::BI__builtin_neon_vset_lane_bf16:
9101 case NEON::BI__builtin_neon_vset_lane_f32:
9102 case NEON::BI__builtin_neon_vsetq_lane_i8:
9103 case NEON::BI__builtin_neon_vsetq_lane_i16:
9104 case NEON::BI__builtin_neon_vsetq_lane_i32:
9105 case NEON::BI__builtin_neon_vsetq_lane_i64:
9106 case NEON::BI__builtin_neon_vsetq_lane_bf16:
9107 case NEON::BI__builtin_neon_vsetq_lane_f32:
9108 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
9110 case NEON::BI__builtin_neon_vsha1h_u32:
9111 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
9113 case NEON::BI__builtin_neon_vsha1cq_u32:
9114 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
9116 case NEON::BI__builtin_neon_vsha1pq_u32:
9117 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
9119 case NEON::BI__builtin_neon_vsha1mq_u32:
9120 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
9123 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
9124 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops,
9130 case clang::ARM::BI_MoveToCoprocessor:
9131 case clang::ARM::BI_MoveToCoprocessor2: {
9132 Function *F = CGM.getIntrinsic(BuiltinID == clang::ARM::BI_MoveToCoprocessor
9133 ? Intrinsic::arm_mcr
9134 : Intrinsic::arm_mcr2);
9135 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
9136 Ops[3], Ops[4], Ops[5]});
9141 assert(HasExtraArg);
9143 std::optional<llvm::APSInt> Result =
9148 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
9149 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
9152 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
9158 bool usgn = Result->getZExtValue() == 1;
9159 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
9162 Function *F = CGM.getIntrinsic(Int, Ty);
9163 return Builder.CreateCall(F, Ops,
"vcvtr");
9168 bool usgn =
Type.isUnsigned();
9169 bool rightShift =
false;
9171 llvm::FixedVectorType *VTy =
9173 getTarget().hasBFloat16Type());
9174 llvm::Type *Ty = VTy;
9184 return EmitCommonNeonBuiltinExpr(
9185 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9186 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
9189 switch (BuiltinID) {
9190 default:
return nullptr;
9191 case NEON::BI__builtin_neon_vld1q_lane_v:
9194 if (VTy->getElementType()->isIntegerTy(64)) {
9196 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9197 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
9198 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
9199 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9201 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
9202 llvm::Type *Tys[] = {Ty, Int8PtrTy};
9203 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
9204 Value *Align = getAlignmentValue32(PtrOp0);
9205 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
9207 int Indices[] = {1 - Lane, Lane};
9208 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
9211 case NEON::BI__builtin_neon_vld1_lane_v: {
9212 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9214 Value *Ld = Builder.CreateLoad(PtrOp0);
9215 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
9217 case NEON::BI__builtin_neon_vqrshrn_n_v:
9219 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9220 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrn_n",
9222 case NEON::BI__builtin_neon_vqrshrun_n_v:
9223 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
9224 Ops,
"vqrshrun_n", 1,
true);
9225 case NEON::BI__builtin_neon_vqshrn_n_v:
9226 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9227 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrn_n",
9229 case NEON::BI__builtin_neon_vqshrun_n_v:
9230 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
9231 Ops,
"vqshrun_n", 1,
true);
9232 case NEON::BI__builtin_neon_vrecpe_v:
9233 case NEON::BI__builtin_neon_vrecpeq_v:
9234 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
9236 case NEON::BI__builtin_neon_vrshrn_n_v:
9237 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
9238 Ops,
"vrshrn_n", 1,
true);
9239 case NEON::BI__builtin_neon_vrsra_n_v:
9240 case NEON::BI__builtin_neon_vrsraq_n_v:
9241 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9242 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9243 Ops[2] = EmitNeonShiftVector(Ops[2], Ty,
true);
9244 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9245 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
9246 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
9247 case NEON::BI__builtin_neon_vsri_n_v:
9248 case NEON::BI__builtin_neon_vsriq_n_v:
9251 case NEON::BI__builtin_neon_vsli_n_v:
9252 case NEON::BI__builtin_neon_vsliq_n_v:
9253 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
9254 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
9256 case NEON::BI__builtin_neon_vsra_n_v:
9257 case NEON::BI__builtin_neon_vsraq_n_v:
9258 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9259 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn,
"vsra_n");
9260 return Builder.CreateAdd(Ops[0], Ops[1]);
9261 case NEON::BI__builtin_neon_vst1q_lane_v:
9264 if (VTy->getElementType()->isIntegerTy(64)) {
9265 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9266 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
9267 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9268 Ops[2] = getAlignmentValue32(PtrOp0);
9269 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
9270 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
9274 case NEON::BI__builtin_neon_vst1_lane_v: {
9275 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9276 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
9277 return Builder.CreateStore(Ops[1],
9280 case NEON::BI__builtin_neon_vtbl1_v:
9281 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
9283 case NEON::BI__builtin_neon_vtbl2_v:
9284 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
9286 case NEON::BI__builtin_neon_vtbl3_v:
9287 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
9289 case NEON::BI__builtin_neon_vtbl4_v:
9290 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
9292 case NEON::BI__builtin_neon_vtbx1_v:
9293 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
9295 case NEON::BI__builtin_neon_vtbx2_v:
9296 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
9298 case NEON::BI__builtin_neon_vtbx3_v:
9299 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
9301 case NEON::BI__builtin_neon_vtbx4_v:
9302 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
9307 template<
typename Integer>
9313 llvm::Type *
T,
bool Unsigned) {
9316 return Unsigned ? Builder.CreateZExt(
V,
T) : Builder.CreateSExt(
V,
T);
9320 uint32_t
Shift,
bool Unsigned) {
9326 unsigned LaneBits = cast<llvm::VectorType>(
V->getType())
9328 ->getPrimitiveSizeInBits();
9329 if (
Shift == LaneBits) {
9334 return llvm::Constant::getNullValue(
V->getType());
9345 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
9346 return Builder.CreateVectorSplat(Elements,
V);
9352 llvm::Type *DestType) {
9365 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9366 return Builder.CreateCall(
9368 {DestType, V->getType()}),
9371 return Builder.CreateBitCast(
V, DestType);
9379 unsigned InputElements =
9380 cast<llvm::FixedVectorType>(
V->getType())->getNumElements();
9381 for (
unsigned i = 0; i < InputElements; i += 2)
9382 Indices.push_back(i + Odd);
9383 return Builder.CreateShuffleVector(
V, Indices);
9389 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
9391 unsigned InputElements =
9392 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9393 for (
unsigned i = 0; i < InputElements; i++) {
9394 Indices.push_back(i);
9395 Indices.push_back(i + InputElements);
9397 return Builder.CreateShuffleVector(V0, V1, Indices);
9400 template<
unsigned HighBit,
unsigned OtherBits>
9404 llvm::Type *
T = cast<llvm::VectorType>(VT)->getElementType();
9405 unsigned LaneBits =
T->getPrimitiveSizeInBits();
9406 uint32_t
Value = HighBit << (LaneBits - 1);
9408 Value |= (1UL << (LaneBits - 1)) - 1;
9409 llvm::Value *Lane = llvm::ConstantInt::get(
T,
Value);
9415 unsigned ReverseWidth) {
9419 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
9420 unsigned Elements = 128 / LaneSize;
9421 unsigned Mask = ReverseWidth / LaneSize - 1;
9422 for (
unsigned i = 0; i < Elements; i++)
9423 Indices.push_back(i ^ Mask);
9424 return Builder.CreateShuffleVector(
V, Indices);
9430 llvm::Triple::ArchType Arch) {
9431 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
9433 unsigned NumVectors;
9436 switch (BuiltinID) {
9437 #include "clang/Basic/arm_mve_builtin_cg.inc"
9448 switch (CustomCodeGenType) {
9450 case CustomCodeGen::VLD24: {
9454 auto MvecCType = E->
getType();
9455 auto MvecLType = ConvertType(MvecCType);
9456 assert(MvecLType->isStructTy() &&
9457 "Return type for vld[24]q should be a struct");
9458 assert(MvecLType->getStructNumElements() == 1 &&
9459 "Return-type struct for vld[24]q should have one element");
9460 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9461 assert(MvecLTypeInner->isArrayTy() &&
9462 "Return-type struct for vld[24]q should contain an array");
9463 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9464 "Array member of return-type struct vld[24]q has wrong length");
9465 auto VecLType = MvecLTypeInner->getArrayElementType();
9467 Tys.push_back(VecLType);
9469 auto Addr = E->
getArg(0);
9470 Ops.push_back(EmitScalarExpr(Addr));
9471 Tys.push_back(ConvertType(Addr->getType()));
9474 Value *LoadResult = Builder.CreateCall(F, Ops);
9475 Value *MvecOut = PoisonValue::get(MvecLType);
9476 for (
unsigned i = 0; i < NumVectors; ++i) {
9477 Value *Vec = Builder.CreateExtractValue(LoadResult, i);
9478 MvecOut = Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9481 if (ReturnValue.isNull())
9484 return Builder.CreateStore(MvecOut, ReturnValue.getAddress());
9487 case CustomCodeGen::VST24: {
9491 auto Addr = E->
getArg(0);
9492 Ops.push_back(EmitScalarExpr(Addr));
9493 Tys.push_back(ConvertType(Addr->getType()));
9496 auto MvecLType = ConvertType(MvecCType);
9497 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
9498 assert(MvecLType->getStructNumElements() == 1 &&
9499 "Data-type struct for vst2q should have one element");
9500 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9501 assert(MvecLTypeInner->isArrayTy() &&
9502 "Data-type struct for vst2q should contain an array");
9503 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9504 "Array member of return-type struct vld[24]q has wrong length");
9505 auto VecLType = MvecLTypeInner->getArrayElementType();
9507 Tys.push_back(VecLType);
9510 EmitAggExpr(E->
getArg(1), MvecSlot);
9511 auto Mvec = Builder.CreateLoad(MvecSlot.
getAddress());
9512 for (
unsigned i = 0; i < NumVectors; i++)
9513 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i}));
9516 Value *ToReturn =
nullptr;
9517 for (
unsigned i = 0; i < NumVectors; i++) {
9518 Ops.push_back(llvm::ConstantInt::get(Int32Ty, i));
9519 ToReturn = Builder.CreateCall(F, Ops);
9525 llvm_unreachable(
"unknown custom codegen type.");
9531 llvm::Triple::ArchType Arch) {
9532 switch (BuiltinID) {
9535 #include "clang/Basic/arm_cde_builtin_cg.inc"
9542 llvm::Triple::ArchType Arch) {
9543 unsigned int Int = 0;
9544 const char *
s =
nullptr;
9546 switch (BuiltinID) {
9549 case NEON::BI__builtin_neon_vtbl1_v:
9550 case NEON::BI__builtin_neon_vqtbl1_v:
9551 case NEON::BI__builtin_neon_vqtbl1q_v:
9552 case NEON::BI__builtin_neon_vtbl2_v:
9553 case NEON::BI__builtin_neon_vqtbl2_v:
9554 case NEON::BI__builtin_neon_vqtbl2q_v:
9555 case NEON::BI__builtin_neon_vtbl3_v:
9556 case NEON::BI__builtin_neon_vqtbl3_v:
9557 case NEON::BI__builtin_neon_vqtbl3q_v:
9558 case NEON::BI__builtin_neon_vtbl4_v:
9559 case NEON::BI__builtin_neon_vqtbl4_v:
9560 case NEON::BI__builtin_neon_vqtbl4q_v:
9562 case NEON::BI__builtin_neon_vtbx1_v:
9563 case NEON::BI__builtin_neon_vqtbx1_v:
9564 case NEON::BI__builtin_neon_vqtbx1q_v:
9565 case NEON::BI__builtin_neon_vtbx2_v:
9566 case NEON::BI__builtin_neon_vqtbx2_v:
9567 case NEON::BI__builtin_neon_vqtbx2q_v:
9568 case NEON::BI__builtin_neon_vtbx3_v:
9569 case NEON::BI__builtin_neon_vqtbx3_v:
9570 case NEON::BI__builtin_neon_vqtbx3q_v:
9571 case NEON::BI__builtin_neon_vtbx4_v:
9572 case NEON::BI__builtin_neon_vqtbx4_v:
9573 case NEON::BI__builtin_neon_vqtbx4q_v:
9581 std::optional<llvm::APSInt> Result =
9596 switch (BuiltinID) {
9597 case NEON::BI__builtin_neon_vtbl1_v: {
9599 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9601 case NEON::BI__builtin_neon_vtbl2_v: {
9603 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9605 case NEON::BI__builtin_neon_vtbl3_v: {
9607 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9609 case NEON::BI__builtin_neon_vtbl4_v: {
9611 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9613 case NEON::BI__builtin_neon_vtbx1_v: {
9616 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9618 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
9619 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
9620 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9622 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9623 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9624 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9626 case NEON::BI__builtin_neon_vtbx2_v: {
9628 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
9630 case NEON::BI__builtin_neon_vtbx3_v: {
9633 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9635 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
9636 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
9638 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9640 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9641 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9642 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9644 case NEON::BI__builtin_neon_vtbx4_v: {
9646 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
9648 case NEON::BI__builtin_neon_vqtbl1_v:
9649 case NEON::BI__builtin_neon_vqtbl1q_v:
9650 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
9651 case NEON::BI__builtin_neon_vqtbl2_v:
9652 case NEON::BI__builtin_neon_vqtbl2q_v: {
9653 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
9654 case NEON::BI__builtin_neon_vqtbl3_v:
9655 case NEON::BI__builtin_neon_vqtbl3q_v:
9656 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
9657 case NEON::BI__builtin_neon_vqtbl4_v:
9658 case NEON::BI__builtin_neon_vqtbl4q_v:
9659 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
9660 case NEON::BI__builtin_neon_vqtbx1_v:
9661 case NEON::BI__builtin_neon_vqtbx1q_v:
9662 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
9663 case NEON::BI__builtin_neon_vqtbx2_v:
9664 case NEON::BI__builtin_neon_vqtbx2q_v:
9665 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
9666 case NEON::BI__builtin_neon_vqtbx3_v:
9667 case NEON::BI__builtin_neon_vqtbx3q_v:
9668 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
9669 case NEON::BI__builtin_neon_vqtbx4_v:
9670 case NEON::BI__builtin_neon_vqtbx4q_v:
9671 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
9683 auto *VTy = llvm::FixedVectorType::get(Int16Ty, 4);
9684 Op = Builder.CreateBitCast(Op, Int16Ty);
9685 Value *
V = PoisonValue::get(VTy);
9686 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
9687 Op = Builder.CreateInsertElement(
V, Op, CI);
9696 case SVETypeFlags::MemEltTyDefault:
9697 return getEltType(TypeFlags);
9698 case SVETypeFlags::MemEltTyInt8:
9699 return Builder.getInt8Ty();
9700 case SVETypeFlags::MemEltTyInt16:
9701 return Builder.getInt16Ty();
9702 case SVETypeFlags::MemEltTyInt32:
9703 return Builder.getInt32Ty();
9704 case SVETypeFlags::MemEltTyInt64:
9705 return Builder.getInt64Ty();
9707 llvm_unreachable(
"Unknown MemEltType");
9713 llvm_unreachable(
"Invalid SVETypeFlag!");
9715 case SVETypeFlags::EltTyInt8:
9716 return Builder.getInt8Ty();
9717 case SVETypeFlags::EltTyInt16:
9718 return Builder.getInt16Ty();
9719 case SVETypeFlags::EltTyInt32:
9720 return Builder.getInt32Ty();
9721 case SVETypeFlags::EltTyInt64:
9722 return Builder.getInt64Ty();
9723 case SVETypeFlags::EltTyInt128:
9724 return Builder.getInt128Ty();
9726 case SVETypeFlags::EltTyFloat16:
9727 return Builder.getHalfTy();
9728 case SVETypeFlags::EltTyFloat32:
9729 return Builder.getFloatTy();
9730 case SVETypeFlags::EltTyFloat64:
9731 return Builder.getDoubleTy();
9733 case SVETypeFlags::EltTyBFloat16:
9734 return Builder.getBFloatTy();
9736 case SVETypeFlags::EltTyBool8:
9737 case SVETypeFlags::EltTyBool16:
9738 case SVETypeFlags::EltTyBool32:
9739 case SVETypeFlags::EltTyBool64:
9740 return Builder.getInt1Ty();
9746 llvm::ScalableVectorType *
9749 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
9751 case SVETypeFlags::EltTyInt8:
9752 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9753 case SVETypeFlags::EltTyInt16:
9754 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9755 case SVETypeFlags::EltTyInt32:
9756 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9757 case SVETypeFlags::EltTyInt64:
9758 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9760 case SVETypeFlags::EltTyBFloat16:
9761 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9762 case SVETypeFlags::EltTyFloat16:
9763 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9764 case SVETypeFlags::EltTyFloat32:
9765 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9766 case SVETypeFlags::EltTyFloat64:
9767 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9769 case SVETypeFlags::EltTyBool8:
9770 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9771 case SVETypeFlags::EltTyBool16:
9772 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9773 case SVETypeFlags::EltTyBool32:
9774 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9775 case SVETypeFlags::EltTyBool64:
9776 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9781 llvm::ScalableVectorType *
9785 llvm_unreachable(
"Invalid SVETypeFlag!");
9787 case SVETypeFlags::EltTyInt8:
9788 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
9789 case SVETypeFlags::EltTyInt16:
9790 return llvm::ScalableVectorType::get(Builder.getInt16Ty(), 8);
9791 case SVETypeFlags::EltTyInt32:
9792 return llvm::ScalableVectorType::get(Builder.getInt32Ty(), 4);
9793 case SVETypeFlags::EltTyInt64:
9794 return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
9796 case SVETypeFlags::EltTyFloat16:
9797 return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
9798 case SVETypeFlags::EltTyBFloat16:
9799 return llvm::ScalableVectorType::get(Builder.getBFloatTy(), 8);
9800 case SVETypeFlags::EltTyFloat32:
9801 return llvm::ScalableVectorType::get(Builder.getFloatTy(), 4);
9802 case SVETypeFlags::EltTyFloat64:
9803 return llvm::ScalableVectorType::get(Builder.getDoubleTy(), 2);
9805 case SVETypeFlags::EltTyBool8:
9806 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9807 case SVETypeFlags::EltTyBool16:
9808 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9809 case SVETypeFlags::EltTyBool32:
9810 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9811 case SVETypeFlags::EltTyBool64:
9812 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9819 CGM.getIntrinsic(Intrinsic::aarch64_sve_ptrue, getSVEPredType(TypeFlags));
9820 return Builder.CreateCall(Ptrue, {Builder.getInt32( 31)});
9827 return llvm::ScalableVectorType::get(EltTy, NumElts);
9833 llvm::ScalableVectorType *VTy) {
9835 if (isa<TargetExtType>(Pred->
getType()) &&
9836 cast<TargetExtType>(Pred->
getType())->getName() ==
"aarch64.svcount")
9839 auto *RTy = llvm::VectorType::get(IntegerType::get(getLLVMContext(), 1), VTy);
9844 llvm::Type *IntrinsicTy;
9845 switch (VTy->getMinNumElements()) {
9847 llvm_unreachable(
"unsupported element count!");
9852 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
9856 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
9857 IntrinsicTy = Pred->
getType();
9861 Function *F = CGM.getIntrinsic(IntID, IntrinsicTy);
9862 Value *C = Builder.CreateCall(F, Pred);
9863 assert(C->getType() == RTy &&
"Unexpected return type!");
9870 auto *ResultTy = getSVEType(TypeFlags);
9871 auto *OverloadedTy =
9872 llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), ResultTy);
9875 if (Ops[1]->getType()->isVectorTy())
9879 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()});
9885 F = CGM.getIntrinsic(IntID, OverloadedTy);
9894 Ops[0] = EmitSVEPredicateCast(
9895 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
9900 if (Ops.size() == 2) {
9901 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9902 Ops.push_back(ConstantInt::get(Int64Ty, 0));
9907 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
9908 unsigned BytesPerElt =
9909 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9910 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9913 Value *Call = Builder.CreateCall(F, Ops);
9917 return TypeFlags.
isZExtReturn() ? Builder.CreateZExt(Call, ResultTy)
9918 : Builder.CreateSExt(Call, ResultTy);
9924 auto *SrcDataTy = getSVEType(TypeFlags);
9925 auto *OverloadedTy =
9926 llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), SrcDataTy);
9930 Ops.insert(Ops.begin(), Ops.pop_back_val());
9933 if (Ops[2]->getType()->isVectorTy())
9937 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()});
9943 F = CGM.getIntrinsic(IntID, OverloadedTy);
9948 if (Ops.size() == 3) {
9949 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9950 Ops.push_back(ConstantInt::get(Int64Ty, 0));
9955 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy);
9964 Ops[1] = EmitSVEPredicateCast(
9965 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
9969 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
9970 unsigned BytesPerElt =
9971 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9972 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
9975 return Builder.CreateCall(F, Ops);
9983 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
9985 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
9988 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy);
9991 if (Ops[1]->getType()->isVectorTy()) {
9992 if (Ops.size() == 3) {
9994 Ops.push_back(ConstantInt::get(Int64Ty, 0));
9997 std::swap(Ops[2], Ops[3]);
10000 llvm::Type *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
10001 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
10002 if (BytesPerElt > 1)
10003 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
10007 Function *F = CGM.getIntrinsic(IntID, OverloadedTy);
10008 return Builder.CreateCall(F, Ops);
10014 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
10018 case Intrinsic::aarch64_sve_ld2_sret:
10019 case Intrinsic::aarch64_sve_ld1_pn_x2:
10020 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
10021 case Intrinsic::aarch64_sve_ld2q_sret:
10024 case Intrinsic::aarch64_sve_ld3_sret:
10025 case Intrinsic::aarch64_sve_ld3q_sret:
10028 case Intrinsic::aarch64_sve_ld4_sret:
10029 case Intrinsic::aarch64_sve_ld1_pn_x4:
10030 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
10031 case Intrinsic::aarch64_sve_ld4q_sret:
10035 llvm_unreachable(
"unknown intrinsic!");
10037 auto RetTy = llvm::VectorType::get(VTy->getElementType(),
10038 VTy->getElementCount() * N);
10040 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy);
10041 Value *BasePtr = Ops[1];
10044 if (Ops.size() > 2)
10045 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
10047 Function *F = CGM.getIntrinsic(IntID, {VTy});
10048 Value *Call = Builder.CreateCall(F, {Predicate, BasePtr});
10049 unsigned MinElts = VTy->getMinNumElements();
10050 Value *
Ret = llvm::PoisonValue::get(RetTy);
10051 for (
unsigned I = 0; I < N; I++) {
10052 Value *Idx = ConstantInt::get(CGM.Int64Ty, I * MinElts);
10053 Value *SRet = Builder.CreateExtractValue(Call, I);
10054 Ret = Builder.CreateInsertVector(RetTy,
Ret, SRet, Idx);
10062 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
10066 case Intrinsic::aarch64_sve_st2:
10067 case Intrinsic::aarch64_sve_st1_pn_x2:
10068 case Intrinsic::aarch64_sve_stnt1_pn_x2:
10069 case Intrinsic::aarch64_sve_st2q:
10072 case Intrinsic::aarch64_sve_st3:
10073 case Intrinsic::aarch64_sve_st3q:
10076 case Intrinsic::aarch64_sve_st4:
10077 case Intrinsic::aarch64_sve_st1_pn_x4:
10078 case Intrinsic::aarch64_sve_stnt1_pn_x4:
10079 case Intrinsic::aarch64_sve_st4q:
10083 llvm_unreachable(
"unknown intrinsic!");
10086 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy);
10087 Value *BasePtr = Ops[1];
10090 if (Ops.size() > (2 + N))
10091 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
10096 for (
unsigned I = Ops.size() - N; I < Ops.size(); ++I)
10097 Operands.push_back(Ops[I]);
10098 Operands.append({Predicate, BasePtr});
10099 Function *F = CGM.getIntrinsic(IntID, { VTy });
10101 return Builder.CreateCall(F, Operands);
10109 unsigned BuiltinID) {
10113 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
10117 Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType());
10118 Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]});
10121 llvm::ScalableVectorType *Ty = getSVEType(TypeFlags);
10122 return EmitSVEReinterpret(Call, Ty);
10127 llvm::Type *OverloadedTy = getSVEType(TypeFlags);
10128 Function *F = CGM.getIntrinsic(BuiltinID, OverloadedTy);
10129 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)});
10134 unsigned BuiltinID) {
10135 auto *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
10137 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10139 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy);
10140 Value *BasePtr = Ops[1];
10143 if (Ops.size() > 3) {
10144 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
10147 Value *PrfOp = Ops.back();
10150 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
10154 llvm::Type *ReturnTy,
10156 unsigned IntrinsicID,
10157 bool IsZExtReturn) {
10159 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
10164 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
10165 llvm::ScalableVectorType *MemoryTy =
nullptr;
10166 llvm::ScalableVectorType *PredTy =
nullptr;
10167 bool IsQuadLoad =
false;
10168 switch (IntrinsicID) {
10169 case Intrinsic::aarch64_sve_ld1uwq:
10170 case Intrinsic::aarch64_sve_ld1udq:
10171 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10172 PredTy = llvm::ScalableVectorType::get(
10173 llvm::Type::getInt1Ty(getLLVMContext()), 1);
10177 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10182 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy);
10183 Value *BasePtr = Ops[1];
10186 if (Ops.size() > 2)
10187 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
10189 BasePtr = Builder.CreateBitCast(BasePtr, MemEltTy->getPointerTo());
10190 Function *F = CGM.getIntrinsic(IntrinsicID, IsQuadLoad ? VectorTy : MemoryTy);
10192 cast<llvm::Instruction>(Builder.CreateCall(F, {Predicate, BasePtr}));
10193 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->
getPointeeType());
10194 CGM.DecorateInstructionWithTBAA(
Load, TBAAInfo);
10199 return IsZExtReturn ? Builder.CreateZExt(
Load, VectorTy)
10200 : Builder.CreateSExt(
Load, VectorTy);
10205 unsigned IntrinsicID) {
10207 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
10212 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
10213 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10215 auto PredTy = MemoryTy;
10216 auto AddrMemoryTy = MemoryTy;
10217 bool IsQuadStore =
false;
10219 switch (IntrinsicID) {
10220 case Intrinsic::aarch64_sve_st1wq:
10221 case Intrinsic::aarch64_sve_st1dq:
10222 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10224 llvm::ScalableVectorType::get(IntegerType::get(getLLVMContext(), 1), 1);
10225 IsQuadStore =
true;
10230 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy);
10231 Value *BasePtr = Ops[1];
10234 if (Ops.size() == 4)
10235 BasePtr = Builder.CreateGEP(AddrMemoryTy, BasePtr, Ops[2]);
10239 IsQuadStore ? Ops.back() : Builder.CreateTrunc(Ops.back(), MemoryTy);
10241 BasePtr = Builder.CreateBitCast(BasePtr, MemEltTy->getPointerTo());
10243 CGM.getIntrinsic(IntrinsicID, IsQuadStore ? VectorTy : MemoryTy);
10245 cast<llvm::Instruction>(Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10246 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->
getPointeeType());
10247 CGM.DecorateInstructionWithTBAA(
Store, TBAAInfo);
10254 Ops[2] = EmitSVEPredicateCast(
10258 NewOps.push_back(Ops[2]);
10260 llvm::Value *BasePtr = Ops[3];
10264 if (Ops.size() == 5) {
10266 CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsb);
10267 llvm::Value *StreamingVectorLengthCall =
10268 Builder.CreateCall(StreamingVectorLength);
10269 llvm::Value *Mulvl =
10270 Builder.CreateMul(StreamingVectorLengthCall, Ops[4],
"mulvl");
10272 BasePtr = Builder.CreateGEP(Int8Ty, Ops[3], Mulvl);
10274 NewOps.push_back(BasePtr);
10275 NewOps.push_back(Ops[0]);
10276 NewOps.push_back(Ops[1]);
10277 Function *F = CGM.getIntrinsic(IntID);
10278 return Builder.CreateCall(F, NewOps);
10284 auto *VecTy = getSVEType(TypeFlags);
10285 Function *F = CGM.getIntrinsic(IntID, VecTy);
10287 Ops[1] = EmitSVEPredicateCast(Ops[1], VecTy);
10289 Ops[2] = EmitSVEPredicateCast(Ops[2], VecTy);
10290 return Builder.CreateCall(F, Ops);
10297 if (Ops.size() == 0)
10298 Ops.push_back(llvm::ConstantInt::get(Int32Ty, 255));
10299 Function *F = CGM.getIntrinsic(IntID, {});
10300 return Builder.CreateCall(F, Ops);
10306 if (Ops.size() == 2)
10307 Ops.push_back(Builder.getInt32(0));
10309 Ops[2] = Builder.CreateIntCast(Ops[2], Int32Ty,
true);
10310 Function *F = CGM.getIntrinsic(IntID, {});
10311 return Builder.CreateCall(F, Ops);
10317 return Builder.CreateVectorSplat(
10318 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10332 return Builder.CreateBitCast(Val, Ty);
10337 auto *SplatZero = Constant::getNullValue(Ty);
10338 Ops.insert(Ops.begin(), SplatZero);
10343 auto *SplatUndef = UndefValue::get(Ty);
10344 Ops.insert(Ops.begin(), SplatUndef);
10349 llvm::Type *ResultType,
10354 llvm::Type *DefaultType = getSVEType(TypeFlags);
10357 return {DefaultType, Ops[1]->getType()};
10360 return {getSVEPredType(TypeFlags), Ops[0]->getType()};
10363 return {Ops[0]->getType(), Ops.back()->getType()};
10365 if (TypeFlags.
isReductionQV() && !ResultType->isScalableTy() &&
10366 ResultType->isVectorTy())
10367 return {ResultType, Ops[1]->getType()};
10370 return {DefaultType};
10377 "Expects TypleFlag isTupleSet or TypeFlags.isTupleSet()");
10379 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue();
10380 auto *SingleVecTy = dyn_cast<llvm::ScalableVectorType>(
10381 TypeFlags.
isTupleSet() ? Ops[2]->getType() : Ty);
10382 Value *Idx = ConstantInt::get(CGM.Int64Ty,
10383 I * SingleVecTy->getMinNumElements());
10386 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx);
10387 return Builder.CreateExtractVector(Ty, Ops[0], Idx);
10393 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
10395 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType());
10396 unsigned MinElts = SrcTy->getMinNumElements();
10397 Value *Call = llvm::PoisonValue::get(Ty);
10398 for (
unsigned I = 0; I < Ops.size(); I++) {
10399 Value *Idx = ConstantInt::get(CGM.Int64Ty, I * MinElts);
10400 Call = Builder.CreateInsertVector(Ty, Call, Ops[I], Idx);
10409 auto *StructTy = dyn_cast<StructType>(Call->getType());
10413 auto *VTy = dyn_cast<ScalableVectorType>(StructTy->getTypeAtIndex(0
U));
10416 unsigned N = StructTy->getNumElements();
10419 bool IsPredTy = VTy->getElementType()->isIntegerTy(1);
10420 unsigned MinElts = IsPredTy ? 16 : VTy->getMinNumElements();
10422 ScalableVectorType *WideVTy =
10423 ScalableVectorType::get(VTy->getElementType(), MinElts * N);
10424 Value *
Ret = llvm::PoisonValue::get(WideVTy);
10425 for (
unsigned I = 0; I < N; ++I) {
10426 Value *SRet = Builder.CreateExtractValue(Call, I);
10427 assert(SRet->
getType() == VTy &&
"Unexpected type for result value");
10428 Value *Idx = ConstantInt::get(CGM.Int64Ty, I * MinElts);
10431 SRet = EmitSVEPredicateCast(
10432 SRet, ScalableVectorType::get(Builder.getInt1Ty(), 16));
10434 Ret = Builder.CreateInsertVector(WideVTy,
Ret, SRet, Idx);
10445 unsigned ICEArguments = 0;
10447 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
10454 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
10455 bool IsICE = ICEArguments & (1 << i);
10461 std::optional<llvm::APSInt> Result =
10463 assert(Result &&
"Expected argument to be a constant");
10468 *Result = Result->extOrTrunc(32);
10469 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), *Result));
10473 if (IsTupleGetOrSet || !isa<ScalableVectorType>(Arg->getType())) {
10474 Ops.push_back(Arg);
10478 auto *VTy = cast<ScalableVectorType>(Arg->getType());
10479 unsigned MinElts = VTy->getMinNumElements();
10480 bool IsPred = VTy->getElementType()->isIntegerTy(1);
10481 unsigned N = (MinElts * VTy->getScalarSizeInBits()) / (IsPred ? 16 : 128);
10484 Ops.push_back(Arg);
10488 for (
unsigned I = 0; I < N; ++I) {
10489 Value *Idx = ConstantInt::get(CGM.Int64Ty, (I * MinElts) / N);
10491 ScalableVectorType::get(VTy->getElementType(), MinElts / N);
10492 Ops.push_back(Builder.CreateExtractVector(NewVTy, Arg, Idx));
10499 llvm::Type *Ty = ConvertType(E->
getType());
10500 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
10501 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
10503 return EmitSVEReinterpret(Val, Ty);
10511 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
10514 return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic,
10516 else if (TypeFlags.
isStore())
10517 return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic);
10519 return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10521 return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10523 return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10525 return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10527 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10529 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10531 return EmitSVETupleSetOrGet(TypeFlags, Ty, Ops);
10533 return EmitSVETupleCreate(TypeFlags, Ty, Ops);
10534 else if (TypeFlags.
isUndef())
10535 return UndefValue::get(Ty);
10536 else if (Builtin->LLVMIntrinsic != 0) {
10537 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
10540 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
10546 Ops.push_back(Builder.getInt32( 31));
10548 Ops.insert(&Ops[1], Builder.getInt32( 31));
10551 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10552 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10553 if (PredTy->getElementType()->isIntegerTy(1))
10554 Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags));
10559 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
10563 std::swap(Ops[1], Ops[2]);
10565 std::swap(Ops[1], Ops[2]);
10568 std::swap(Ops[1], Ops[2]);
10571 std::swap(Ops[1], Ops[3]);
10574 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
10575 llvm::Type *OpndTy = Ops[1]->getType();
10576 auto *SplatZero = Constant::getNullValue(OpndTy);
10577 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10580 Function *F = CGM.getIntrinsic(Builtin->LLVMIntrinsic,
10581 getSVEOverloadTypes(TypeFlags, Ty, Ops));
10582 Value *Call = Builder.CreateCall(F, Ops);
10585 if (
auto PredTy = dyn_cast<llvm::VectorType>(Call->getType()))
10586 if (PredTy->getScalarType()->isIntegerTy(1))
10587 Call = EmitSVEPredicateCast(Call, cast<llvm::ScalableVectorType>(Ty));
10589 return FormSVEBuiltinResult(Call);
10592 switch (BuiltinID) {
10596 case SVE::BI__builtin_sve_svreinterpret_b: {
10598 llvm::TargetExtType::get(getLLVMContext(),
"aarch64.svcount");
10600 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10601 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
10603 case SVE::BI__builtin_sve_svreinterpret_c: {
10605 llvm::TargetExtType::get(getLLVMContext(),
"aarch64.svcount");
10607 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10608 return Builder.CreateCall(CastToSVCountF, Ops[0]);
10611 case SVE::BI__builtin_sve_svpsel_lane_b8:
10612 case SVE::BI__builtin_sve_svpsel_lane_b16:
10613 case SVE::BI__builtin_sve_svpsel_lane_b32:
10614 case SVE::BI__builtin_sve_svpsel_lane_b64:
10615 case SVE::BI__builtin_sve_svpsel_lane_c8:
10616 case SVE::BI__builtin_sve_svpsel_lane_c16:
10617 case SVE::BI__builtin_sve_svpsel_lane_c32:
10618 case SVE::BI__builtin_sve_svpsel_lane_c64: {
10619 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
10620 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->
getName() ==
10621 "aarch64.svcount")) &&
10622 "Unexpected TargetExtType");
10624 llvm::TargetExtType::get(getLLVMContext(),
"aarch64.svcount");
10626 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10628 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10630 auto OverloadedTy = getSVEType(
SVETypeFlags(Builtin->TypeModifier));
10631 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_psel, OverloadedTy);
10632 llvm::Value *Ops0 =
10633 IsSVCount ? Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
10634 llvm::Value *Ops1 = EmitSVEPredicateCast(Ops[1], OverloadedTy);
10635 llvm::Value *PSel = Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
10636 return IsSVCount ? Builder.CreateCall(CastToSVCountF, PSel) : PSel;
10638 case SVE::BI__builtin_sve_svmov_b_z: {
10641 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
10642 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_and_z, OverloadedTy);
10643 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
10646 case SVE::BI__builtin_sve_svnot_b_z: {
10649 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
10650 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_eor_z, OverloadedTy);
10651 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
10654 case SVE::BI__builtin_sve_svmovlb_u16:
10655 case SVE::BI__builtin_sve_svmovlb_u32:
10656 case SVE::BI__builtin_sve_svmovlb_u64:
10657 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
10659 case SVE::BI__builtin_sve_svmovlb_s16:
10660 case SVE::BI__builtin_sve_svmovlb_s32:
10661 case SVE::BI__builtin_sve_svmovlb_s64:
10662 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
10664 case SVE::BI__builtin_sve_svmovlt_u16:
10665 case SVE::BI__builtin_sve_svmovlt_u32:
10666 case SVE::BI__builtin_sve_svmovlt_u64:
10667 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
10669 case SVE::BI__builtin_sve_svmovlt_s16:
10670 case SVE::BI__builtin_sve_svmovlt_s32:
10671 case SVE::BI__builtin_sve_svmovlt_s64:
10672 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
10674 case SVE::BI__builtin_sve_svpmullt_u16:
10675 case SVE::BI__builtin_sve_svpmullt_u64:
10676 case SVE::BI__builtin_sve_svpmullt_n_u16:
10677 case SVE::BI__builtin_sve_svpmullt_n_u64:
10678 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
10680 case SVE::BI__builtin_sve_svpmullb_u16:
10681 case SVE::BI__builtin_sve_svpmullb_u64:
10682 case SVE::BI__builtin_sve_svpmullb_n_u16:
10683 case SVE::BI__builtin_sve_svpmullb_n_u64:
10684 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
10686 case SVE::BI__builtin_sve_svdup_n_b8:
10687 case SVE::BI__builtin_sve_svdup_n_b16:
10688 case SVE::BI__builtin_sve_svdup_n_b32:
10689 case SVE::BI__builtin_sve_svdup_n_b64: {
10691 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
10692 llvm::ScalableVectorType *OverloadedTy = getSVEType(TypeFlags);
10693 Value *
Dup = EmitSVEDupX(CmpNE, OverloadedTy);
10694 return EmitSVEPredicateCast(
Dup, cast<llvm::ScalableVectorType>(Ty));
10697 case SVE::BI__builtin_sve_svdupq_n_b8:
10698 case SVE::BI__builtin_sve_svdupq_n_b16:
10699 case SVE::BI__builtin_sve_svdupq_n_b32:
10700 case SVE::BI__builtin_sve_svdupq_n_b64:
10701 case SVE::BI__builtin_sve_svdupq_n_u8:
10702 case SVE::BI__builtin_sve_svdupq_n_s8:
10703 case SVE::BI__builtin_sve_svdupq_n_u64:
10704 case SVE::BI__builtin_sve_svdupq_n_f64:
10705 case SVE::BI__builtin_sve_svdupq_n_s64:
10706 case SVE::BI__builtin_sve_svdupq_n_u16:
10707 case SVE::BI__builtin_sve_svdupq_n_f16:
10708 case SVE::BI__builtin_sve_svdupq_n_bf16:
10709 case SVE::BI__builtin_sve_svdupq_n_s16:
10710 case SVE::BI__builtin_sve_svdupq_n_u32:
10711 case SVE::BI__builtin_sve_svdupq_n_f32:
10712 case SVE::BI__builtin_sve_svdupq_n_s32: {
10715 unsigned NumOpnds = Ops.size();
10718 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
10723 llvm::Type *EltTy = Ops[0]->getType();
10725 EltTy = IntegerType::get(getLLVMContext(),
SVEBitsPerBlock / NumOpnds);
10728 for (
unsigned I = 0; I < NumOpnds; ++I)
10729 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy));
10730 Value *Vec = BuildVector(VecOps);
10733 Value *InsertSubVec = Builder.CreateInsertVector(
10734 OverloadedTy, PoisonValue::get(OverloadedTy), Vec, Builder.getInt64(0));
10737 CGM.getIntrinsic(Intrinsic::aarch64_sve_dupq_lane, OverloadedTy);
10739 Builder.CreateCall(F, {InsertSubVec, Builder.getInt64(0)});
10745 Value *Pred = EmitSVEAllTruePred(TypeFlags);
10748 F = CGM.getIntrinsic(NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
10749 : Intrinsic::aarch64_sve_cmpne_wide,
10751 Value *Call = Builder.CreateCall(
10752 F, {Pred, DupQLane, EmitSVEDupX(Builder.getInt64(0))});
10753 return EmitSVEPredicateCast(Call, cast<llvm::ScalableVectorType>(Ty));
10756 case SVE::BI__builtin_sve_svpfalse_b:
10757 return ConstantInt::getFalse(Ty);
10759 case SVE::BI__builtin_sve_svpfalse_c: {
10760 auto SVBoolTy = ScalableVectorType::get(Builder.getInt1Ty(), 16);
10762 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, Ty);
10763 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
10766 case SVE::BI__builtin_sve_svlen_bf16:
10767 case SVE::BI__builtin_sve_svlen_f16:
10768 case SVE::BI__builtin_sve_svlen_f32:
10769 case SVE::BI__builtin_sve_svlen_f64:
10770 case SVE::BI__builtin_sve_svlen_s8:
10771 case SVE::BI__builtin_sve_svlen_s16:
10772 case SVE::BI__builtin_sve_svlen_s32:
10773 case SVE::BI__builtin_sve_svlen_s64:
10774 case SVE::BI__builtin_sve_svlen_u8:
10775 case SVE::BI__builtin_sve_svlen_u16:
10776 case SVE::BI__builtin_sve_svlen_u32:
10777 case SVE::BI__builtin_sve_svlen_u64: {
10779 auto VTy = cast<llvm::VectorType>(getSVEType(TF));
10781 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
10783 Function *F = CGM.getIntrinsic(Intrinsic::vscale, Ty);
10784 return Builder.CreateMul(NumEls, Builder.CreateCall(F));
10787 case SVE::BI__builtin_sve_svtbl2_u8:
10788 case SVE::BI__builtin_sve_svtbl2_s8:
10789 case SVE::BI__builtin_sve_svtbl2_u16:
10790 case SVE::BI__builtin_sve_svtbl2_s16:
10791 case SVE::BI__builtin_sve_svtbl2_u32:
10792 case SVE::BI__builtin_sve_svtbl2_s32:
10793 case SVE::BI__builtin_sve_svtbl2_u64:
10794 case SVE::BI__builtin_sve_svtbl2_s64:
10795 case SVE::BI__builtin_sve_svtbl2_f16:
10796 case SVE::BI__builtin_sve_svtbl2_bf16:
10797 case SVE::BI__builtin_sve_svtbl2_f32:
10798 case SVE::BI__builtin_sve_svtbl2_f64: {
10800 auto VTy = cast<llvm::ScalableVectorType>(getSVEType(TF));
10801 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, VTy);
10802 return Builder.CreateCall(F, Ops);
10805 case SVE::BI__builtin_sve_svset_neonq_s8:
10806 case SVE::BI__builtin_sve_svset_neonq_s16:
10807 case SVE::BI__builtin_sve_svset_neonq_s32:
10808 case SVE::BI__builtin_sve_svset_neonq_s64:
10809 case SVE::BI__builtin_sve_svset_neonq_u8:
10810 case SVE::BI__builtin_sve_svset_neonq_u16:
10811 case SVE::BI__builtin_sve_svset_neonq_u32:
10812 case SVE::BI__builtin_sve_svset_neonq_u64:
10813 case SVE::BI__builtin_sve_svset_neonq_f16:
10814 case SVE::BI__builtin_sve_svset_neonq_f32:
10815 case SVE::BI__builtin_sve_svset_neonq_f64:
10816 case SVE::BI__builtin_sve_svset_neonq_bf16: {
10817 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0));
10820 case SVE::BI__builtin_sve_svget_neonq_s8:
10821 case SVE::BI__builtin_sve_svget_neonq_s16:
10822 case SVE::BI__builtin_sve_svget_neonq_s32:
10823 case SVE::BI__builtin_sve_svget_neonq_s64:
10824 case SVE::BI__builtin_sve_svget_neonq_u8:
10825 case SVE::BI__builtin_sve_svget_neonq_u16:
10826 case SVE::BI__builtin_sve_svget_neonq_u32:
10827 case SVE::BI__builtin_sve_svget_neonq_u64:
10828 case SVE::BI__builtin_sve_svget_neonq_f16:
10829 case SVE::BI__builtin_sve_svget_neonq_f32:
10830 case SVE::BI__builtin_sve_svget_neonq_f64:
10831 case SVE::BI__builtin_sve_svget_neonq_bf16: {
10832 return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0));
10835 case SVE::BI__builtin_sve_svdup_neonq_s8:
10836 case SVE::BI__builtin_sve_svdup_neonq_s16:
10837 case SVE::BI__builtin_sve_svdup_neonq_s32:
10838 case SVE::BI__builtin_sve_svdup_neonq_s64:
10839 case SVE::BI__builtin_sve_svdup_neonq_u8:
10840 case SVE::BI__builtin_sve_svdup_neonq_u16:
10841 case SVE::BI__builtin_sve_svdup_neonq_u32:
10842 case SVE::BI__builtin_sve_svdup_neonq_u64:
10843 case SVE::BI__builtin_sve_svdup_neonq_f16:
10844 case SVE::BI__builtin_sve_svdup_neonq_f32:
10845 case SVE::BI__builtin_sve_svdup_neonq_f64:
10846 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
10847 Value *
Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
10848 Builder.getInt64(0));
10849 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
10850 {
Insert, Builder.getInt64(0)});
10861 switch (BuiltinID) {
10864 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
10867 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
10868 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
10871 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
10872 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
10878 for (
unsigned I = 0; I < MultiVec; ++I)
10879 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
10889 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
10892 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10894 return EmitSMEReadWrite(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10895 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
10896 BuiltinID == SME::BI__builtin_sme_svzero_za)
10897 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10898 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
10899 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
10900 BuiltinID == SME::BI__builtin_sme_svldr_za ||
10901 BuiltinID == SME::BI__builtin_sme_svstr_za)
10902 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10908 if (Builtin->LLVMIntrinsic == 0)
10912 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10913 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10914 if (PredTy->getElementType()->isIntegerTy(1))
10915 Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags));
10919 ? CGM.getIntrinsic(Builtin->LLVMIntrinsic)
10920 : CGM.getIntrinsic(Builtin->LLVMIntrinsic, {getSVEType(TypeFlags)});
10921 Value *Call = Builder.CreateCall(F, Ops);
10923 return FormSVEBuiltinResult(Call);
10928 llvm::Triple::ArchType Arch) {
10931 return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
10935 return EmitAArch64SMEBuiltinExpr(BuiltinID, E);
10937 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
10938 return EmitAArch64CpuSupports(E);
10940 unsigned HintID =
static_cast<unsigned>(-1);
10941 switch (BuiltinID) {
10943 case clang::AArch64::BI__builtin_arm_nop:
10946 case clang::AArch64::BI__builtin_arm_yield:
10947 case clang::AArch64::BI__yield:
10950 case clang::AArch64::BI__builtin_arm_wfe:
10951 case clang::AArch64::BI__wfe:
10954 case clang::AArch64::BI__builtin_arm_wfi:
10955 case clang::AArch64::BI__wfi:
10958 case clang::AArch64::BI__builtin_arm_sev:
10959 case clang::AArch64::BI__sev:
10962 case clang::AArch64::BI__builtin_arm_sevl:
10963 case clang::AArch64::BI__sevl:
10968 if (HintID !=
static_cast<unsigned>(-1)) {
10969 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
10970 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
10973 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
10974 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break);
10975 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
10976 return Builder.CreateCall(F, Builder.CreateZExt(Arg, CGM.Int32Ty));
10979 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
10981 CallInst *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction(
10982 llvm::FunctionType::get(StructType::get(CGM.Int64Ty, CGM.Int64Ty), {},
10984 "__arm_sme_state"));
10985 auto Attrs = AttributeList().addFnAttribute(getLLVMContext(),
10986 "aarch64_pstate_sm_compatible");
10987 CI->setAttributes(Attrs);
10988 CI->setCallingConv(
10990 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
10991 Builder.CreateStore(Builder.CreateExtractValue(CI, 0),
10993 return Builder.CreateStore(Builder.CreateExtractValue(CI, 1),
10997 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
10998 assert((getContext().getTypeSize(E->
getType()) == 32) &&
10999 "rbit of unusual size!");
11000 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11001 return Builder.CreateCall(
11002 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
11004 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
11005 assert((getContext().getTypeSize(E->
getType()) == 64) &&
11006 "rbit of unusual size!");
11007 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11008 return Builder.CreateCall(
11009 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
11012 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
11013 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
11014 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11015 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
11016 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(
false)});
11017 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
11018 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
11022 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
11023 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11024 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
11027 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
11028 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11029 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
11033 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
11034 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
11035 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11036 llvm::Type *Ty = Arg->getType();
11037 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
11041 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
11042 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
11043 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11044 llvm::Type *Ty = Arg->getType();
11045 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
11049 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
11050 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
11051 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11052 llvm::Type *Ty = Arg->getType();
11053 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
11057 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
11058 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
11059 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11060 llvm::Type *Ty = Arg->getType();
11061 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
11065 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
11066 assert((getContext().getTypeSize(E->
getType()) == 32) &&
11067 "__jcvt of unusual size!");
11068 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
11069 return Builder.CreateCall(
11070 CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
11073 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
11074 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
11075 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
11076 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
11077 llvm::Value *MemAddr = EmitScalarExpr(E->
getArg(0));
11078 llvm::Value *ValPtr = EmitScalarExpr(E->
getArg(1));
11080 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
11083 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ld64b);
11084 llvm::Value *Val = Builder.CreateCall(F, MemAddr);
11085 llvm::Value *ToRet;
11086 for (
size_t i = 0; i < 8; i++) {
11087 llvm::Value *ValOffsetPtr =
11088 Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
11091 ToRet = Builder.CreateStore(Builder.CreateExtractValue(Val, i), Addr);
11098 Args.push_back(MemAddr);
11099 for (
size_t i = 0; i < 8; i++) {
11100 llvm::Value *ValOffsetPtr =
11101 Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
11104 Args.push_back(Builder.CreateLoad(Addr));
11107 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
11108 ? Intrinsic::aarch64_st64b
11109 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
11110 ? Intrinsic::aarch64_st64bv
11111 : Intrinsic::aarch64_st64bv0);
11112 Function *F = CGM.getIntrinsic(Intr);
11113 return Builder.CreateCall(F, Args);
11117 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
11118 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
11120 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
11121 ? Intrinsic::aarch64_rndr
11122 : Intrinsic::aarch64_rndrrs);
11123 Function *F = CGM.getIntrinsic(Intr);
11124 llvm::Value *Val = Builder.CreateCall(F);
11125 Value *RandomValue = Builder.CreateExtractValue(Val, 0);
11126 Value *Status = Builder.CreateExtractValue(Val, 1);
11129 Builder.CreateStore(RandomValue, MemAddress);
11130 Status = Builder.CreateZExt(Status, Int32Ty);
11134 if (BuiltinID == clang::AArch64::BI__clear_cache) {
11135 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
11138 for (
unsigned i = 0; i < 2; i++)
11139 Ops[i] = EmitScalarExpr(E->
getArg(i));
11140 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->
getType());
11141 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
11142 StringRef Name = FD->
getName();
11143 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
11146 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11147 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
11148 getContext().getTypeSize(E->
getType()) == 128) {
11150 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
11151 ? Intrinsic::aarch64_ldaxp
11152 : Intrinsic::aarch64_ldxp);
11155 Value *Val = Builder.CreateCall(F, LdPtr,
"ldxp");
11157 Value *Val0 = Builder.CreateExtractValue(Val, 1);
11158 Value *Val1 = Builder.CreateExtractValue(Val, 0);
11159 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
11160 Val0 = Builder.CreateZExt(Val0, Int128Ty);
11161 Val1 = Builder.CreateZExt(Val1, Int128Ty);
11163 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
11164 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
11165 Val = Builder.CreateOr(Val, Val1);
11166 return Builder.CreateBitCast(Val, ConvertType(E->
getType()));
11167 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11168 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
11172 llvm::Type *RealResTy = ConvertType(Ty);
11173 llvm::Type *IntTy =
11174 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
11175 llvm::Type *PtrTy = IntTy->getPointerTo();
11176 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
11179 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
11180 ? Intrinsic::aarch64_ldaxr
11181 : Intrinsic::aarch64_ldxr,
11183 CallInst *Val = Builder.CreateCall(F, LoadAddr,
"ldxr");
11185 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
11187 if (RealResTy->isPointerTy())
11188 return Builder.CreateIntToPtr(Val, RealResTy);
11190 llvm::Type *IntResTy = llvm::IntegerType::get(
11191 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
11192 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
11196 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11197 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
11198 getContext().getTypeSize(E->
getArg(0)->
getType()) == 128) {
11200 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
11201 ? Intrinsic::aarch64_stlxp
11202 : Intrinsic::aarch64_stxp);
11203 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
11209 llvm::Value *Val = Builder.CreateLoad(Tmp);
11211 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
11212 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
11214 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
11217 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11218 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
11223 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
11224 getContext().getTypeSize(Ty));
11225 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
11227 if (StoreVal->
getType()->isPointerTy())
11228 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
11230 llvm::Type *IntTy = llvm::IntegerType::get(
11232 CGM.getDataLayout().getTypeSizeInBits(StoreVal->
getType()));
11233 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
11234 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
11238 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
11239 ? Intrinsic::aarch64_stlxr
11240 : Intrinsic::aarch64_stxr,
11242 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
11244 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
11248 if (BuiltinID == clang::AArch64::BI__getReg) {
11251 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11254 LLVMContext &Context = CGM.getLLVMContext();
11257 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
11258 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11259 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11261 llvm::Function *F =
11262 CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
11263 return Builder.CreateCall(F, Metadata);
11266 if (BuiltinID == clang::AArch64::BI__break) {
11269 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11271 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::aarch64_break);
11272 return Builder.CreateCall(F, {EmitScalarExpr(E->
getArg(0))});
11275 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
11276 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
11277 return Builder.CreateCall(F);
11280 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
11281 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11282 llvm::SyncScope::SingleThread);
11286 switch (BuiltinID) {
11287 case clang::AArch64::BI__builtin_arm_crc32b:
11288 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
11289 case clang::AArch64::BI__builtin_arm_crc32cb:
11290 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
11291 case clang::AArch64::BI__builtin_arm_crc32h:
11292 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
11293 case clang::AArch64::BI__builtin_arm_crc32ch:
11294 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
11295 case clang::AArch64::BI__builtin_arm_crc32w:
11296 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
11297 case clang::AArch64::BI__builtin_arm_crc32cw:
11298 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
11299 case clang::AArch64::BI__builtin_arm_crc32d:
11300 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
11301 case clang::AArch64::BI__builtin_arm_crc32cd:
11302 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
11305 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11308 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
11310 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11311 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
11313 return Builder.CreateCall(F, {Arg0, Arg1});
11317 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
11321 Dst = Builder.CreatePointerCast(Dst, Int8PtrTy);
11322 Val = Builder.CreateTrunc(Val, Int8Ty);
11323 Size = Builder.CreateIntCast(Size, Int64Ty,
false);
11324 return Builder.CreateCall(
11325 CGM.getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11330 switch (BuiltinID) {
11331 case clang::AArch64::BI__builtin_arm_irg:
11332 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
11333 case clang::AArch64::BI__builtin_arm_addg:
11334 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
11335 case clang::AArch64::BI__builtin_arm_gmi:
11336 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
11337 case clang::AArch64::BI__builtin_arm_ldg:
11338 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
11339 case clang::AArch64::BI__builtin_arm_stg:
11340 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
11341 case clang::AArch64::BI__builtin_arm_subp:
11342 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
11345 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11346 llvm::Type *
T = ConvertType(E->
getType());
11348 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11352 Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
11353 Mask = Builder.CreateZExt(Mask, Int64Ty);
11354 Value *RV = Builder.CreateCall(
11355 CGM.getIntrinsic(MTEIntrinsicID), {Pointer, Mask});
11356 return Builder.CreatePointerCast(RV,
T);
11358 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11362 Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
11363 TagOffset = Builder.CreateZExt(TagOffset, Int64Ty);
11364 Value *RV = Builder.CreateCall(
11365 CGM.getIntrinsic(MTEIntrinsicID), {Pointer, TagOffset});
11366 return Builder.CreatePointerCast(RV,
T);
11368 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11370 Value *ExcludedMask = EmitScalarExpr(E->
getArg(1));
11372 ExcludedMask = Builder.CreateZExt(ExcludedMask, Int64Ty);
11373 Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
11374 return Builder.CreateCall(
11375 CGM.getIntrinsic(MTEIntrinsicID), {Pointer, ExcludedMask});
11380 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11381 Value *TagAddress = EmitScalarExpr(E->
getArg(0));
11382 TagAddress = Builder.CreatePointerCast(TagAddress, Int8PtrTy);
11383 Value *RV = Builder.CreateCall(
11384 CGM.getIntrinsic(MTEIntrinsicID), {TagAddress, TagAddress});
11385 return Builder.CreatePointerCast(RV,
T);
11390 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11391 Value *TagAddress = EmitScalarExpr(E->
getArg(0));
11392 TagAddress = Builder.CreatePointerCast(TagAddress, Int8PtrTy);
11393 return Builder.CreateCall(
11394 CGM.getIntrinsic(MTEIntrinsicID), {TagAddress, TagAddress});
11396 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11399 PointerA = Builder.CreatePointerCast(PointerA, Int8PtrTy);
11400 PointerB = Builder.CreatePointerCast(PointerB, Int8PtrTy);
11401 return Builder.CreateCall(
11402 CGM.getIntrinsic(MTEIntrinsicID), {PointerA, PointerB});
11406 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11407 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11408 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11409 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11410 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
11411 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
11412 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
11413 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
11416 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11417 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11418 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11419 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
11422 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11423 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
11425 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11426 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
11428 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11429 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
11431 llvm::Type *ValueType;
11432 llvm::Type *RegisterType = Int64Ty;
11434 ValueType = Int32Ty;
11435 }
else if (Is128Bit) {
11436 llvm::Type *Int128Ty =
11437 llvm::IntegerType::getInt128Ty(CGM.getLLVMContext());
11438 ValueType = Int128Ty;
11439 RegisterType = Int128Ty;
11440 }
else if (IsPointerBuiltin) {
11441 ValueType = VoidPtrTy;
11443 ValueType = Int64Ty;
11450 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
11451 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
11452 LLVMContext &Context = CGM.getLLVMContext();
11457 std::string SysRegStr;
11458 llvm::raw_string_ostream(SysRegStr) <<
11459 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
11460 ((SysReg >> 11) & 7) <<
":" <<
11461 ((SysReg >> 7) & 15) <<
":" <<
11462 ((SysReg >> 3) & 15) <<
":" <<
11465 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
11466 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11467 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11469 llvm::Type *RegisterType = Int64Ty;
11470 llvm::Type *Types[] = { RegisterType };
11472 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
11473 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
11475 return Builder.CreateCall(F, Metadata);
11478 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
11479 llvm::Value *ArgValue = EmitScalarExpr(E->
getArg(1));
11481 return Builder.CreateCall(F, { Metadata, ArgValue });
11484 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
11485 llvm::Function *F =
11486 CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
11487 return Builder.CreateCall(F);
11490 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
11491 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
11492 return Builder.CreateCall(F);
11495 if (BuiltinID == clang::AArch64::BI__mulh ||
11496 BuiltinID == clang::AArch64::BI__umulh) {
11497 llvm::Type *ResType = ConvertType(E->
getType());
11498 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
11500 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
11502 Builder.CreateIntCast(EmitScalarExpr(E->
getArg(0)), Int128Ty, IsSigned);
11504 Builder.CreateIntCast(EmitScalarExpr(E->
getArg(1)), Int128Ty, IsSigned);
11506 Value *MulResult, *HigherBits;
11508 MulResult = Builder.CreateNSWMul(LHS, RHS);
11509 HigherBits = Builder.CreateAShr(MulResult, 64);
11511 MulResult = Builder.CreateNUWMul(LHS, RHS);
11512 HigherBits = Builder.CreateLShr(MulResult, 64);
11514 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11519 if (BuiltinID == AArch64::BI__writex18byte ||
11520 BuiltinID == AArch64::BI__writex18word ||
11521 BuiltinID == AArch64::BI__writex18dword ||
11522 BuiltinID == AArch64::BI__writex18qword) {
11524 LLVMContext &Context = CGM.getLLVMContext();
11525 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11526 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11527 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11528 llvm::Function *F =
11529 CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
11530 llvm::Value *X18 = Builder.CreateCall(F, Metadata);
11531 X18 = Builder.CreateIntToPtr(X18, Int8PtrTy);
11535 Value *Ptr = Builder.CreateGEP(Int8Ty, X18,
Offset);
11541 if (BuiltinID == AArch64::BI__readx18byte ||
11542 BuiltinID == AArch64::BI__readx18word ||
11543 BuiltinID == AArch64::BI__readx18dword ||
11544 BuiltinID == AArch64::BI__readx18qword) {
11545 llvm::Type *IntTy = ConvertType(E->
getType());
11548 LLVMContext &Context = CGM.getLLVMContext();
11549 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11550 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11551 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11552 llvm::Function *F =
11553 CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
11554 llvm::Value *X18 = Builder.CreateCall(F, Metadata);
11555 X18 = Builder.CreateIntToPtr(X18, Int8PtrTy);
11559 Value *Ptr = Builder.CreateGEP(Int8Ty, X18,
Offset);
11564 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
11565 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
11566 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
11567 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
11569 llvm::Type *RetTy = ConvertType(E->
getType());
11570 return Builder.CreateBitCast(Arg, RetTy);
11573 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11574 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11575 BuiltinID == AArch64::BI_CountLeadingZeros ||
11576 BuiltinID == AArch64::BI_CountLeadingZeros64) {
11578 llvm::Type *ArgType = Arg->
getType();
11580 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11581 BuiltinID == AArch64::BI_CountLeadingOnes64)
11582 Arg = Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
11584 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
11585 Value *Result = Builder.CreateCall(F, {Arg, Builder.getInt1(
false)});
11587 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11588 BuiltinID == AArch64::BI_CountLeadingZeros64)
11589 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
11593 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
11594 BuiltinID == AArch64::BI_CountLeadingSigns64) {
11597 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
11598 ? CGM.getIntrinsic(Intrinsic::aarch64_cls)
11599 : CGM.getIntrinsic(Intrinsic::aarch64_cls64);
11601 Value *Result = Builder.CreateCall(F, Arg,
"cls");
11602 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
11603 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
11607 if (BuiltinID == AArch64::BI_CountOneBits ||
11608 BuiltinID == AArch64::BI_CountOneBits64) {
11610 llvm::Type *ArgType = ArgValue->
getType();
11611 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
11613 Value *Result = Builder.CreateCall(F, ArgValue);
11614 if (BuiltinID == AArch64::BI_CountOneBits64)
11615 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
11619 if (BuiltinID == AArch64::BI__prefetch) {
11621 Value *RW = llvm::ConstantInt::get(Int32Ty, 0);
11622 Value *Locality = ConstantInt::get(Int32Ty, 3);
11623 Value *
Data = llvm::ConstantInt::get(Int32Ty, 1);
11625 return Builder.CreateCall(F, {
Address, RW, Locality,
Data});
11630 if (std::optional<MSVCIntrin> MsvcIntId =
11632 return EmitMSVCBuiltinExpr(*MsvcIntId, E);
11636 return P.first == BuiltinID;
11639 BuiltinID = It->second;
11643 unsigned ICEArguments = 0;
11645 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
11650 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
11652 switch (BuiltinID) {
11653 case NEON::BI__builtin_neon_vld1_v:
11654 case NEON::BI__builtin_neon_vld1q_v:
11655 case NEON::BI__builtin_neon_vld1_dup_v:
11656 case NEON::BI__builtin_neon_vld1q_dup_v:
11657 case NEON::BI__builtin_neon_vld1_lane_v:
11658 case NEON::BI__builtin_neon_vld1q_lane_v:
11659 case NEON::BI__builtin_neon_vst1_v:
11660 case NEON::BI__builtin_neon_vst1q_v:
11661 case NEON::BI__builtin_neon_vst1_lane_v:
11662 case NEON::BI__builtin_neon_vst1q_lane_v:
11663 case NEON::BI__builtin_neon_vldap1_lane_s64:
11664 case NEON::BI__builtin_neon_vldap1q_lane_s64:
11665 case NEON::BI__builtin_neon_vstl1_lane_s64:
11666 case NEON::BI__builtin_neon_vstl1q_lane_s64:
11674 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E));
11684 assert(Result &&
"SISD intrinsic should have been handled");
11690 if (std::optional<llvm::APSInt> Result =
11695 bool usgn =
Type.isUnsigned();
11696 bool quad =
Type.isQuad();
11699 switch (BuiltinID) {
11701 case NEON::BI__builtin_neon_vabsh_f16:
11702 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11703 return EmitNeonCall(CGM.getIntrinsic(
Intrinsic::fabs, HalfTy), Ops,
"vabs");
11704 case NEON::BI__builtin_neon_vaddq_p128: {
11706 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
11707 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11708 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11709 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
11710 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
11711 return Builder.CreateBitCast(Ops[0], Int128Ty);
11713 case NEON::BI__builtin_neon_vldrq_p128: {
11714 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
11716 return Builder.CreateAlignedLoad(Int128Ty, Ptr,
11719 case NEON::BI__builtin_neon_vstrq_p128: {
11720 Value *Ptr = Ops[0];
11721 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->
getArg(1)), Ptr);
11723 case NEON::BI__builtin_neon_vcvts_f32_u32:
11724 case NEON::BI__builtin_neon_vcvtd_f64_u64:
11727 case NEON::BI__builtin_neon_vcvts_f32_s32:
11728 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
11729 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11730 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
11731 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
11732 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
11733 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
11735 return Builder.CreateUIToFP(Ops[0], FTy);
11736 return Builder.CreateSIToFP(Ops[0], FTy);
11738 case NEON::BI__builtin_neon_vcvth_f16_u16:
11739 case NEON::BI__builtin_neon_vcvth_f16_u32:
11740 case NEON::BI__builtin_neon_vcvth_f16_u64:
11743 case NEON::BI__builtin_neon_vcvth_f16_s16:
11744 case NEON::BI__builtin_neon_vcvth_f16_s32:
11745 case NEON::BI__builtin_neon_vcvth_f16_s64: {
11746 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11747 llvm::Type *FTy = HalfTy;
11749 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
11751 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
11755 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
11757 return Builder.CreateUIToFP(Ops[0], FTy);
11758 return Builder.CreateSIToFP(Ops[0], FTy);
11760 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11761 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11762 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11763 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11764 case NEON::BI__builtin_neon_vcvth_u16_f16:
11765 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11766 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11767 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11768 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11769 case NEON::BI__builtin_neon_vcvth_s16_f16: {
11771 llvm::Type* InTy = Int32Ty;
11772 llvm::Type* FTy = HalfTy;
11773 llvm::Type *Tys[2] = {InTy, FTy};
11774 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11775 switch (BuiltinID) {
11776 default: llvm_unreachable(
"missing builtin ID in switch!");
11777 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11778 Int = Intrinsic::aarch64_neon_fcvtau;
break;
11779 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11780 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
11781 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11782 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
11783 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11784 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
11785 case NEON::BI__builtin_neon_vcvth_u16_f16:
11786 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
11787 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11788 Int = Intrinsic::aarch64_neon_fcvtas;
break;
11789 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11790 Int = Intrinsic::aarch64_neon_fcvtms;
break;
11791 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11792 Int = Intrinsic::aarch64_neon_fcvtns;
break;
11793 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11794 Int = Intrinsic::aarch64_neon_fcvtps;
break;
11795 case NEON::BI__builtin_neon_vcvth_s16_f16:
11796 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
11798 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvt");
11799 return Builder.CreateTrunc(Ops[0], Int16Ty);
11801 case NEON::BI__builtin_neon_vcaleh_f16:
11802 case NEON::BI__builtin_neon_vcalth_f16:
11803 case NEON::BI__builtin_neon_vcageh_f16:
11804 case NEON::BI__builtin_neon_vcagth_f16: {
11806 llvm::Type* InTy = Int32Ty;
11807 llvm::Type* FTy = HalfTy;
11808 llvm::Type *Tys[2] = {InTy, FTy};
11809 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
11810 switch (BuiltinID) {
11811 default: llvm_unreachable(
"missing builtin ID in switch!");
11812 case NEON::BI__builtin_neon_vcageh_f16:
11813 Int = Intrinsic::aarch64_neon_facge;
break;
11814 case NEON::BI__builtin_neon_vcagth_f16:
11815 Int = Intrinsic::aarch64_neon_facgt;
break;
11816 case NEON::BI__builtin_neon_vcaleh_f16:
11817 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
11818 case NEON::BI__builtin_neon_vcalth_f16:
11819 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
11821 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"facg");
11822 return Builder.CreateTrunc(Ops[0], Int16Ty);
11824 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11825 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
11827 llvm::Type* InTy = Int32Ty;
11828 llvm::Type* FTy = HalfTy;
11829 llvm::Type *Tys[2] = {InTy, FTy};
11830 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
11831 switch (BuiltinID) {
11832 default: llvm_unreachable(
"missing builtin ID in switch!");
11833 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11834 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
11835 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
11836 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
11838 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvth_n");
11839 return Builder.CreateTrunc(Ops[0], Int16Ty);
11841 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11842 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
11844 llvm::Type* FTy = HalfTy;
11845 llvm::Type* InTy = Int32Ty;
11846 llvm::Type *Tys[2] = {FTy, InTy};
11847 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
11848 switch (BuiltinID) {
11849 default: llvm_unreachable(
"missing builtin ID in switch!");
11850 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11851 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
11852 Ops[0] = Builder.CreateSExt(Ops[0], InTy,
"sext");
11854 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
11855 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
11856 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
11859 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvth_n");
11861 case NEON::BI__builtin_neon_vpaddd_s64: {
11862 auto *Ty = llvm::FixedVectorType::get(Int64Ty, 2);
11865 Vec = Builder.CreateBitCast(Vec, Ty,
"v2i64");
11866 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
11867 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
11868 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11869 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11871 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
11873 case NEON::BI__builtin_neon_vpaddd_f64: {
11874 auto *Ty = llvm::FixedVectorType::get(DoubleTy, 2);
11877 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f64");
11878 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
11879 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
11880 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11881 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11883 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11885 case NEON::BI__builtin_neon_vpadds_f32: {
11886 auto *Ty = llvm::FixedVectorType::get(FloatTy, 2);
11889 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f32");
11890 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
11891 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
11892 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11893 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11895 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11897 case NEON::BI__builtin_neon_vceqzd_s64:
11898 case NEON::BI__builtin_neon_vceqzd_f64:
11899 case NEON::BI__builtin_neon_vceqzs_f32:
11900 case NEON::BI__builtin_neon_vceqzh_f16:
11901 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11902 return EmitAArch64CompareBuiltinExpr(
11904 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
11905 case NEON::BI__builtin_neon_vcgezd_s64:
11906 case NEON::BI__builtin_neon_vcgezd_f64:
11907 case NEON::BI__builtin_neon_vcgezs_f32:
11908 case NEON::BI__builtin_neon_vcgezh_f16:
11909 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11910 return EmitAArch64CompareBuiltinExpr(
11912 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
11913 case NEON::BI__builtin_neon_vclezd_s64:
11914 case NEON::BI__builtin_neon_vclezd_f64:
11915 case NEON::BI__builtin_neon_vclezs_f32:
11916 case NEON::BI__builtin_neon_vclezh_f16:
11917 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11918 return EmitAArch64CompareBuiltinExpr(
11920 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
11921 case NEON::BI__builtin_neon_vcgtzd_s64:
11922 case NEON::BI__builtin_neon_vcgtzd_f64:
11923 case NEON::BI__builtin_neon_vcgtzs_f32:
11924 case NEON::BI__builtin_neon_vcgtzh_f16:
11925 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11926 return EmitAArch64CompareBuiltinExpr(
11928 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
11929 case NEON::BI__builtin_neon_vcltzd_s64:
11930 case NEON::BI__builtin_neon_vcltzd_f64:
11931 case NEON::BI__builtin_neon_vcltzs_f32:
11932 case NEON::BI__builtin_neon_vcltzh_f16:
11933 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11934 return EmitAArch64CompareBuiltinExpr(
11936 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
11938 case NEON::BI__builtin_neon_vceqzd_u64: {
11939 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
11940 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
11942 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
11943 return Builder.CreateSExt(Ops[0], Int64Ty,
"vceqzd");
11945 case NEON::BI__builtin_neon_vceqd_f64:
11946 case NEON::BI__builtin_neon_vcled_f64:
11947 case NEON::BI__builtin_neon_vcltd_f64:
11948 case NEON::BI__builtin_neon_vcged_f64:
11949 case NEON::BI__builtin_neon_vcgtd_f64: {
11950 llvm::CmpInst::Predicate
P;
11951 switch (BuiltinID) {
11952 default: llvm_unreachable(
"missing builtin ID in switch!");
11953 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11954 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
11955 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
11956 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
11957 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
11959 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
11960 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
11961 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
11962 if (
P == llvm::FCmpInst::FCMP_OEQ)
11963 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11965 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11966 return Builder.CreateSExt(Ops[0], Int64Ty,
"vcmpd");
11968 case NEON::BI__builtin_neon_vceqs_f32:
11969 case NEON::BI__builtin_neon_vcles_f32:
11970 case NEON::BI__builtin_neon_vclts_f32:
11971 case NEON::BI__builtin_neon_vcges_f32:
11972 case NEON::BI__builtin_neon_vcgts_f32: {
11973 llvm::CmpInst::Predicate
P;
11974 switch (BuiltinID) {
11975 default: llvm_unreachable(
"missing builtin ID in switch!");
11976 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11977 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
11978 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
11979 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
11980 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
11982 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
11983 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
11984 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
11985 if (
P == llvm::FCmpInst::FCMP_OEQ)
11986 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11988 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11989 return Builder.CreateSExt(Ops[0], Int32Ty,
"vcmpd");
11991 case NEON::BI__builtin_neon_vceqh_f16:
11992 case NEON::BI__builtin_neon_vcleh_f16:
11993 case NEON::BI__builtin_neon_vclth_f16:
11994 case NEON::BI__builtin_neon_vcgeh_f16:
11995 case NEON::BI__builtin_neon_vcgth_f16: {
11996 llvm::CmpInst::Predicate
P;
11997 switch (BuiltinID) {
11998 default: llvm_unreachable(
"missing builtin ID in switch!");
11999 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12000 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
12001 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
12002 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
12003 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
12005 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12006 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
12007 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
12008 if (
P == llvm::FCmpInst::FCMP_OEQ)
12009 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12011 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12012 return Builder.CreateSExt(Ops[0], Int16Ty,
"vcmpd");
12014 case NEON::BI__builtin_neon_vceqd_s64:
12015 case NEON::BI__builtin_neon_vceqd_u64:
12016 case NEON::BI__builtin_neon_vcgtd_s64:
12017 case NEON::BI__builtin_neon_vcgtd_u64:
12018 case NEON::BI__builtin_neon_vcltd_s64:
12019 case NEON::BI__builtin_neon_vcltd_u64:
12020 case NEON::BI__builtin_neon_vcged_u64:
12021 case NEON::BI__builtin_neon_vcged_s64:
12022 case NEON::BI__builtin_neon_vcled_u64:
12023 case NEON::BI__builtin_neon_vcled_s64: {
12024 llvm::CmpInst::Predicate
P;
12025 switch (BuiltinID) {
12026 default: llvm_unreachable(
"missing builtin ID in switch!");
12027 case NEON::BI__builtin_neon_vceqd_s64:
12028 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
12029 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
12030 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
12031 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
12032 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
12033 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
12034 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
12035 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
12036 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
12038 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12039 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
12040 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
12041 Ops[0] = Builder.CreateICmp(
P, Ops[0], Ops[1]);
12042 return Builder.CreateSExt(Ops[0], Int64Ty,
"vceqd");
12044 case NEON::BI__builtin_neon_vtstd_s64:
12045 case NEON::BI__builtin_neon_vtstd_u64: {
12046 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12047 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
12048 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
12049 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
12050 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
12051 llvm::Constant::getNullValue(Int64Ty));
12052 return Builder.CreateSExt(Ops[0], Int64Ty,
"vtstd");
12054 case NEON::BI__builtin_neon_vset_lane_i8:
12055 case NEON::BI__builtin_neon_vset_lane_i16:
12056 case NEON::BI__builtin_neon_vset_lane_i32:
12057 case NEON::BI__builtin_neon_vset_lane_i64:
12058 case NEON::BI__builtin_neon_vset_lane_bf16:
12059 case NEON::BI__builtin_neon_vset_lane_f32:
12060 case NEON::BI__builtin_neon_vsetq_lane_i8:
12061 case NEON::BI__builtin_neon_vsetq_lane_i16:
12062 case NEON::BI__builtin_neon_vsetq_lane_i32:
12063 case NEON::BI__builtin_neon_vsetq_lane_i64:
12064 case NEON::BI__builtin_neon_vsetq_lane_bf16:
12065 case NEON::BI__builtin_neon_vsetq_lane_f32:
12066 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
12067 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12068 case NEON::BI__builtin_neon_vset_lane_f64:
12071 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1));
12072 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
12073 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12074 case NEON::BI__builtin_neon_vsetq_lane_f64:
12077 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2));
12078 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
12079 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12081 case NEON::BI__builtin_neon_vget_lane_i8:
12082 case NEON::BI__builtin_neon_vdupb_lane_i8:
12084 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 8));
12085 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12087 case NEON::BI__builtin_neon_vgetq_lane_i8:
12088 case NEON::BI__builtin_neon_vdupb_laneq_i8:
12090 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 16));
12091 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12093 case NEON::BI__builtin_neon_vget_lane_i16:
12094 case NEON::BI__builtin_neon_vduph_lane_i16:
12096 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 4));
12097 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12099 case NEON::BI__builtin_neon_vgetq_lane_i16:
12100 case NEON::BI__builtin_neon_vduph_laneq_i16:
12102 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 8));
12103 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12105 case NEON::BI__builtin_neon_vget_lane_i32:
12106 case NEON::BI__builtin_neon_vdups_lane_i32:
12108 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 2));
12109 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12111 case NEON::BI__builtin_neon_vdups_lane_f32:
12113 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2));
12114 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12116 case NEON::BI__builtin_neon_vgetq_lane_i32:
12117 case NEON::BI__builtin_neon_vdups_laneq_i32:
12119 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4));
12120 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12122 case NEON::BI__builtin_neon_vget_lane_i64:
12123 case NEON::BI__builtin_neon_vdupd_lane_i64:
12125 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 1));
12126 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12128 case NEON::BI__builtin_neon_vdupd_lane_f64:
12130 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1));
12131 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12133 case NEON::BI__builtin_neon_vgetq_lane_i64:
12134 case NEON::BI__builtin_neon_vdupd_laneq_i64:
12136 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
12137 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12139 case NEON::BI__builtin_neon_vget_lane_f32:
12141 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2));
12142 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12144 case NEON::BI__builtin_neon_vget_lane_f64:
12146 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1));
12147 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12149 case NEON::BI__builtin_neon_vgetq_lane_f32:
12150 case NEON::BI__builtin_neon_vdups_laneq_f32:
12152 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 4));
12153 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12155 case NEON::BI__builtin_neon_vgetq_lane_f64:
12156 case NEON::BI__builtin_neon_vdupd_laneq_f64:
12158 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 2));
12159 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12161 case NEON::BI__builtin_neon_vaddh_f16:
12162 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12163 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
12164 case NEON::BI__builtin_neon_vsubh_f16:
12165 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12166 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
12167 case NEON::BI__builtin_neon_vmulh_f16:
12168 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12169 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
12170 case NEON::BI__builtin_neon_vdivh_f16:
12171 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12172 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
12173 case NEON::BI__builtin_neon_vfmah_f16:
12176 *
this,
Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
12177 {EmitScalarExpr(E->
getArg(1)), EmitScalarExpr(E->
getArg(2)), Ops[0]});
12178 case NEON::BI__builtin_neon_vfmsh_f16: {
12179 Value*
Neg = Builder.CreateFNeg(EmitScalarExpr(E->
getArg(1)),
"vsubh");
12183 *
this,
Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
12184 {
Neg, EmitScalarExpr(E->
getArg(2)), Ops[0]});
12186 case NEON::BI__builtin_neon_vaddd_s64:
12187 case NEON::BI__builtin_neon_vaddd_u64:
12188 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->
getArg(1)),
"vaddd");
12189 case NEON::BI__builtin_neon_vsubd_s64:
12190 case NEON::BI__builtin_neon_vsubd_u64:
12191 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->
getArg(1)),
"vsubd");
12192 case NEON::BI__builtin_neon_vqdmlalh_s16:
12193 case NEON::BI__builtin_neon_vqdmlslh_s16: {
12195 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
12196 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->
getArg(2))));
12197 auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
12198 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
12199 ProductOps,
"vqdmlXl");
12200 Constant *CI = ConstantInt::get(SizeTy, 0);
12201 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12203 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
12204 ? Intrinsic::aarch64_neon_sqadd
12205 : Intrinsic::aarch64_neon_sqsub;
12206 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops,
"vqdmlXl");
12208 case NEON::BI__builtin_neon_vqshlud_n_s64: {
12209 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12210 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
12211 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
12214 case NEON::BI__builtin_neon_vqshld_n_u64:
12215 case NEON::BI__builtin_neon_vqshld_n_s64: {
12216 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
12217 ? Intrinsic::aarch64_neon_uqshl
12218 : Intrinsic::aarch64_neon_sqshl;
12219 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12220 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
12221 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops,
"vqshl_n");
12223 case NEON::BI__builtin_neon_vrshrd_n_u64:
12224 case NEON::BI__builtin_neon_vrshrd_n_s64: {
12225 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
12226 ? Intrinsic::aarch64_neon_urshl
12227 : Intrinsic::aarch64_neon_srshl;
12228 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12229 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12230 Ops[1] = ConstantInt::get(Int64Ty, -SV);
12231 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops,
"vrshr_n");
12233 case NEON::BI__builtin_neon_vrsrad_n_u64:
12234 case NEON::BI__builtin_neon_vrsrad_n_s64: {
12235 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
12236 ? Intrinsic::aarch64_neon_urshl
12237 : Intrinsic::aarch64_neon_srshl;
12238 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
12239 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->
getArg(2))));
12240 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
12241 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12242 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
12244 case NEON::BI__builtin_neon_vshld_n_s64:
12245 case NEON::BI__builtin_neon_vshld_n_u64: {
12246 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
12247 return Builder.CreateShl(
12248 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()),
"shld_n");
12250 case NEON::BI__builtin_neon_vshrd_n_s64: {
12251 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
12252 return Builder.CreateAShr(
12254 Amt->getZExtValue())),
12257 case NEON::BI__builtin_neon_vshrd_n_u64: {
12258 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
12259 uint64_t ShiftAmt = Amt->getZExtValue();
12261 if (ShiftAmt == 64)
12262 return ConstantInt::get(Int64Ty, 0);
12263 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
12266 case NEON::BI__builtin_neon_vsrad_n_s64: {
12267 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(2)));
12268 Ops[1] = Builder.CreateAShr(
12270 Amt->getZExtValue())),
12272 return Builder.CreateAdd(Ops[0], Ops[1]);
12274 case NEON::BI__builtin_neon_vsrad_n_u64: {
12275 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(2)));
12276 uint64_t ShiftAmt = Amt->getZExtValue();
12279 if (ShiftAmt == 64)
12281 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
12283 return Builder.CreateAdd(Ops[0], Ops[1]);
12285 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
12286 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
12287 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
12288 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
12289 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->
getArg(3)),
12292 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
12293 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
12294 auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
12295 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
12296 ProductOps,
"vqdmlXl");
12297 Constant *CI = ConstantInt::get(SizeTy, 0);
12298 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12301 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
12302 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
12303 ? Intrinsic::aarch64_neon_sqadd
12304 : Intrinsic::aarch64_neon_sqsub;
12305 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops,
"vqdmlXl");
12307 case NEON::BI__builtin_neon_vqdmlals_s32:
12308 case NEON::BI__builtin_neon_vqdmlsls_s32: {
12310 ProductOps.push_back(Ops[1]);
12311 ProductOps.push_back(EmitScalarExpr(E->
getArg(2)));
12313 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
12314 ProductOps,
"vqdmlXl");
12316 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
12317 ? Intrinsic::aarch64_neon_sqadd
12318 : Intrinsic::aarch64_neon_sqsub;
12319 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops,
"vqdmlXl");
12321 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
12322 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
12323 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
12324 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
12325 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->
getArg(3)),
12328 ProductOps.push_back(Ops[1]);
12329 ProductOps.push_back(Ops[2]);
12331 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
12332 ProductOps,
"vqdmlXl");
12335 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
12336 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
12337 ? Intrinsic::aarch64_neon_sqadd
12338 : Intrinsic::aarch64_neon_sqsub;
12339 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops,
"vqdmlXl");
12341 case NEON::BI__builtin_neon_vget_lane_bf16:
12342 case NEON::BI__builtin_neon_vduph_lane_bf16:
12343 case NEON::BI__builtin_neon_vduph_lane_f16: {
12344 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12347 case NEON::BI__builtin_neon_vgetq_lane_bf16:
12348 case NEON::BI__builtin_neon_vduph_laneq_bf16:
12349 case NEON::BI__builtin_neon_vduph_laneq_f16: {
12350 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
12354 case clang::AArch64::BI_InterlockedAdd:
12355 case clang::AArch64::BI_InterlockedAdd64: {
12358 AtomicRMWInst *RMWI =
12360 llvm::AtomicOrdering::SequentiallyConsistent);
12361 return Builder.CreateAdd(RMWI, Val);
12366 llvm::Type *Ty = VTy;
12376 return EmitCommonNeonBuiltinExpr(
12377 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12378 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
12385 switch (BuiltinID) {
12386 default:
return nullptr;
12387 case NEON::BI__builtin_neon_vbsl_v:
12388 case NEON::BI__builtin_neon_vbslq_v: {
12389 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
12390 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
12391 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
12392 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
12394 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
12395 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
12396 Ops[0] = Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
12397 return Builder.CreateBitCast(Ops[0], Ty);
12399 case NEON::BI__builtin_neon_vfma_lane_v:
12400 case NEON::BI__builtin_neon_vfmaq_lane_v: {
12403 Value *Addend = Ops[0];
12404 Value *Multiplicand = Ops[1];
12405 Value *LaneSource = Ops[2];
12406 Ops[0] = Multiplicand;
12407 Ops[1] = LaneSource;
12411 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
12412 ? llvm::FixedVectorType::get(VTy->getElementType(),
12413 VTy->getNumElements() / 2)
12415 llvm::Constant *cst = cast<Constant>(Ops[3]);
12416 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12417 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
12418 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
12421 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12423 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"fmla");
12425 case NEON::BI__builtin_neon_vfma_laneq_v: {
12426 auto *VTy = cast<llvm::FixedVectorType>(Ty);
12428 if (VTy && VTy->getElementType() == DoubleTy) {
12429 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
12430 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
12431 llvm::FixedVectorType *VTy =
12433 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
12434 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12438 DoubleTy, {Ops[1], Ops[2], Ops[0]});
12439 return Builder.CreateBitCast(Result, Ty);
12441 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12442 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12444 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12445 VTy->getNumElements() * 2);
12446 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
12447 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12448 cast<ConstantInt>(Ops[3]));
12449 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
12452 *
this,
Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12453 {Ops[2], Ops[1], Ops[0]});
12455 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
12456 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12457 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12459 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
12460 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
12462 *
this,
Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12463 {Ops[2], Ops[1], Ops[0]});
12465 case NEON::BI__builtin_neon_vfmah_lane_f16:
12466 case NEON::BI__builtin_neon_vfmas_lane_f32:
12467 case NEON::BI__builtin_neon_vfmah_laneq_f16:
12468 case NEON::BI__builtin_neon_vfmas_laneq_f32:
12469 case NEON::BI__builtin_neon_vfmad_lane_f64:
12470 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
12471 Ops.push_back(EmitScalarExpr(E->
getArg(3)));
12473 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12475 *
this,
Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12476 {Ops[1], Ops[2], Ops[0]});
12478 case NEON::BI__builtin_neon_vmull_v:
12480 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12481 if (
Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
12482 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmull");
12483 case NEON::BI__builtin_neon_vmax_v:
12484 case NEON::BI__builtin_neon_vmaxq_v:
12486 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12487 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
12488 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmax");
12489 case NEON::BI__builtin_neon_vmaxh_f16: {
12490 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12491 Int = Intrinsic::aarch64_neon_fmax;
12492 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmax");
12494 case NEON::BI__builtin_neon_vmin_v:
12495 case NEON::BI__builtin_neon_vminq_v:
12497 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12498 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
12499 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmin");
12500 case NEON::BI__builtin_neon_vminh_f16: {
12501 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12502 Int = Intrinsic::aarch64_neon_fmin;
12503 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmin");
12505 case NEON::BI__builtin_neon_vabd_v:
12506 case NEON::BI__builtin_neon_vabdq_v:
12508 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12509 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
12510 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vabd");
12511 case NEON::BI__builtin_neon_vpadal_v:
12512 case NEON::BI__builtin_neon_vpadalq_v: {
12513 unsigned ArgElts = VTy->getNumElements();
12514 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12515 unsigned BitWidth = EltTy->getBitWidth();
12516 auto *ArgTy = llvm::FixedVectorType::get(
12517 llvm::IntegerType::get(getLLVMContext(), BitWidth / 2), 2 * ArgElts);
12518 llvm::Type* Tys[2] = { VTy, ArgTy };
12519 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12521 TmpOps.push_back(Ops[1]);
12522 Function *F = CGM.getIntrinsic(Int, Tys);
12523 llvm::Value *tmp = EmitNeonCall(F, TmpOps,
"vpadal");
12524 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
12525 return Builder.CreateAdd(tmp, addend);
12527 case NEON::BI__builtin_neon_vpmin_v:
12528 case NEON::BI__builtin_neon_vpminq_v:
12530 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12531 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
12532 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmin");
12533 case NEON::BI__builtin_neon_vpmax_v:
12534 case NEON::BI__builtin_neon_vpmaxq_v:
12536 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12537 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
12538 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmax");
12539 case NEON::BI__builtin_neon_vminnm_v:
12540 case NEON::BI__builtin_neon_vminnmq_v:
12541 Int = Intrinsic::aarch64_neon_fminnm;
12542 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vminnm");
12543 case NEON::BI__builtin_neon_vminnmh_f16:
12544 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12545 Int = Intrinsic::aarch64_neon_fminnm;
12546 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vminnm");
12547 case NEON::BI__builtin_neon_vmaxnm_v:
12548 case NEON::BI__builtin_neon_vmaxnmq_v:
12549 Int = Intrinsic::aarch64_neon_fmaxnm;
12550 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmaxnm");
12551 case NEON::BI__builtin_neon_vmaxnmh_f16:
12552 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12553 Int = Intrinsic::aarch64_neon_fmaxnm;
12554 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmaxnm");
12555 case NEON::BI__builtin_neon_vrecpss_f32: {
12556 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12557 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
12560 case NEON::BI__builtin_neon_vrecpsd_f64:
12561 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12562 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
12564 case NEON::BI__builtin_neon_vrecpsh_f16:
12565 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
12566 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, HalfTy),
12568 case NEON::BI__builtin_neon_vqshrun_n_v:
12569 Int = Intrinsic::aarch64_neon_sqshrun;
12570 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrun_n");
12571 case NEON::BI__builtin_neon_vqrshrun_n_v:
12572 Int = Intrinsic::aarch64_neon_sqrshrun;
12573 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrun_n");
12574 case NEON::BI__builtin_neon_vqshrn_n_v:
12575 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
12576 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrn_n");
12577 case NEON::BI__builtin_neon_vrshrn_n_v:
12578 Int = Intrinsic::aarch64_neon_rshrn;
12579 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrshrn_n");
12580 case NEON::BI__builtin_neon_vqrshrn_n_v:
12581 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
12582 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrn_n");
12583 case NEON::BI__builtin_neon_vrndah_f16: {
12584 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12585 Int = Builder.getIsFPConstrained()
12586 ? Intrinsic::experimental_constrained_round
12588 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrnda");
12590 case NEON::BI__builtin_neon_vrnda_v:
12591 case NEON::BI__builtin_neon_vrndaq_v: {
12592 Int = Builder.getIsFPConstrained()
12593 ? Intrinsic::experimental_constrained_round
12595 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrnda");
12597 case NEON::BI__builtin_neon_vrndih_f16: {
12598 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12599 Int = Builder.getIsFPConstrained()
12600 ? Intrinsic::experimental_constrained_nearbyint
12602 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndi");
12604 case NEON::BI__builtin_neon_vrndmh_f16: {
12605 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12606 Int = Builder.getIsFPConstrained()
12607 ? Intrinsic::experimental_constrained_floor
12609 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndm");
12611 case NEON::BI__builtin_neon_vrndm_v:
12612 case NEON::BI__builtin_neon_vrndmq_v: {
12613 Int = Builder.getIsFPConstrained()
12614 ? Intrinsic::experimental_constrained_floor
12616 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndm");
12618 case NEON::BI__builtin_neon_vrndnh_f16: {
12619 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12620 Int = Builder.getIsFPConstrained()
12621 ? Intrinsic::experimental_constrained_roundeven
12622 : Intrinsic::roundeven;
12623 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndn");
12625 case NEON::BI__builtin_neon_vrndn_v:
12626 case NEON::BI__builtin_neon_vrndnq_v: {
12627 Int = Builder.getIsFPConstrained()
12628 ? Intrinsic::experimental_constrained_roundeven
12629 : Intrinsic::roundeven;
12630 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndn");
12632 case NEON::BI__builtin_neon_vrndns_f32: {
12633 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12634 Int = Builder.getIsFPConstrained()
12635 ? Intrinsic::experimental_constrained_roundeven
12636 : Intrinsic::roundeven;
12637 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops,
"vrndn");
12639 case NEON::BI__builtin_neon_vrndph_f16: {
12640 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12641 Int = Builder.getIsFPConstrained()
12642 ? Intrinsic::experimental_constrained_ceil
12644 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndp");
12646 case NEON::BI__builtin_neon_vrndp_v:
12647 case NEON::BI__builtin_neon_vrndpq_v: {
12648 Int = Builder.getIsFPConstrained()
12649 ? Intrinsic::experimental_constrained_ceil
12651 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndp");
12653 case NEON::BI__builtin_neon_vrndxh_f16: {
12654 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12655 Int = Builder.getIsFPConstrained()
12656 ? Intrinsic::experimental_constrained_rint
12658 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndx");
12660 case NEON::BI__builtin_neon_vrndx_v:
12661 case NEON::BI__builtin_neon_vrndxq_v: {
12662 Int = Builder.getIsFPConstrained()
12663 ? Intrinsic::experimental_constrained_rint
12665 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndx");
12667 case NEON::BI__builtin_neon_vrndh_f16: {
12668 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12669 Int = Builder.getIsFPConstrained()
12670 ? Intrinsic::experimental_constrained_trunc
12672 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndz");
12674 case NEON::BI__builtin_neon_vrnd32x_f32:
12675 case NEON::BI__builtin_neon_vrnd32xq_f32:
12676 case NEON::BI__builtin_neon_vrnd32x_f64:
12677 case NEON::BI__builtin_neon_vrnd32xq_f64: {
12678 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12679 Int = Intrinsic::aarch64_neon_frint32x;
12680 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrnd32x");
12682 case NEON::BI__builtin_neon_vrnd32z_f32:
12683 case NEON::BI__builtin_neon_vrnd32zq_f32:
12684 case NEON::BI__builtin_neon_vrnd32z_f64:
12685 case NEON::BI__builtin_neon_vrnd32zq_f64: {
12686 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12687 Int = Intrinsic::aarch64_neon_frint32z;
12688 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrnd32z");
12690 case NEON::BI__builtin_neon_vrnd64x_f32:
12691 case NEON::BI__builtin_neon_vrnd64xq_f32:
12692 case NEON::BI__builtin_neon_vrnd64x_f64:
12693 case NEON::BI__builtin_neon_vrnd64xq_f64: {
12694 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12695 Int = Intrinsic::aarch64_neon_frint64x;
12696 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrnd64x");
12698 case NEON::BI__builtin_neon_vrnd64z_f32:
12699 case NEON::BI__builtin_neon_vrnd64zq_f32:
12700 case NEON::BI__builtin_neon_vrnd64z_f64:
12701 case NEON::BI__builtin_neon_vrnd64zq_f64: {
12702 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12703 Int = Intrinsic::aarch64_neon_frint64z;
12704 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrnd64z");
12706 case NEON::BI__builtin_neon_vrnd_v:
12707 case NEON::BI__builtin_neon_vrndq_v: {
12708 Int = Builder.getIsFPConstrained()
12709 ? Intrinsic::experimental_constrained_trunc
12711 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndz");
12713 case NEON::BI__builtin_neon_vcvt_f64_v:
12714 case NEON::BI__builtin_neon_vcvtq_f64_v:
12715 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12717 return usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
12718 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
12719 case NEON::BI__builtin_neon_vcvt_f64_f32: {
12721 "unexpected vcvt_f64_f32 builtin");
12723 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
12725 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
12727 case NEON::BI__builtin_neon_vcvt_f32_f64: {
12729 "unexpected vcvt_f32_f64 builtin");
12731 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
12733 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
12735 case NEON::BI__builtin_neon_vcvt_s32_v:
12736 case NEON::BI__builtin_neon_vcvt_u32_v:
12737 case NEON::BI__builtin_neon_vcvt_s64_v:
12738 case NEON::BI__builtin_neon_vcvt_u64_v:
12739 case NEON::BI__builtin_neon_vcvt_s16_f16:
12740 case NEON::BI__builtin_neon_vcvt_u16_f16:
12741 case NEON::BI__builtin_neon_vcvtq_s32_v:
12742 case NEON::BI__builtin_neon_vcvtq_u32_v:
12743 case NEON::BI__builtin_neon_vcvtq_s64_v:
12744 case NEON::BI__builtin_neon_vcvtq_u64_v:
12745 case NEON::BI__builtin_neon_vcvtq_s16_f16:
12746 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
12748 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
12750 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtz");
12752 case NEON::BI__builtin_neon_vcvta_s16_f16:
12753 case NEON::BI__builtin_neon_vcvta_u16_f16:
12754 case NEON::BI__builtin_neon_vcvta_s32_v:
12755 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
12756 case NEON::BI__builtin_neon_vcvtaq_s32_v:
12757 case NEON::BI__builtin_neon_vcvta_u32_v:
12758 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
12759 case NEON::BI__builtin_neon_vcvtaq_u32_v:
12760 case NEON::BI__builtin_neon_vcvta_s64_v:
12761 case NEON::BI__builtin_neon_vcvtaq_s64_v:
12762 case NEON::BI__builtin_neon_vcvta_u64_v:
12763 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
12764 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
12766 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvta");
12768 case NEON::BI__builtin_neon_vcvtm_s16_f16:
12769 case NEON::BI__builtin_neon_vcvtm_s32_v:
12770 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
12771 case NEON::BI__builtin_neon_vcvtmq_s32_v:
12772 case NEON::BI__builtin_neon_vcvtm_u16_f16:
12773 case NEON::BI__builtin_neon_vcvtm_u32_v:
12774 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
12775 case NEON::BI__builtin_neon_vcvtmq_u32_v:
12776 case NEON::BI__builtin_neon_vcvtm_s64_v:
12777 case NEON::BI__builtin_neon_vcvtmq_s64_v:
12778 case NEON::BI__builtin_neon_vcvtm_u64_v:
12779 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
12780 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
12782 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtm");
12784 case NEON::BI__builtin_neon_vcvtn_s16_f16:
12785 case NEON::BI__builtin_neon_vcvtn_s32_v:
12786 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
12787 case NEON::BI__builtin_neon_vcvtnq_s32_v:
12788 case NEON::BI__builtin_neon_vcvtn_u16_f16:
12789 case NEON::BI__builtin_neon_vcvtn_u32_v:
12790 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
12791 case NEON::BI__builtin_neon_vcvtnq_u32_v:
12792 case NEON::BI__builtin_neon_vcvtn_s64_v:
12793 case NEON::BI__builtin_neon_vcvtnq_s64_v:
12794 case NEON::BI__builtin_neon_vcvtn_u64_v:
12795 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
12796 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
12798 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtn");
12800 case NEON::BI__builtin_neon_vcvtp_s16_f16:
12801 case NEON::BI__builtin_neon_vcvtp_s32_v:
12802 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
12803 case NEON::BI__builtin_neon_vcvtpq_s32_v:
12804 case NEON::BI__builtin_neon_vcvtp_u16_f16:
12805 case NEON::BI__builtin_neon_vcvtp_u32_v:
12806 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
12807 case NEON::BI__builtin_neon_vcvtpq_u32_v:
12808 case NEON::BI__builtin_neon_vcvtp_s64_v:
12809 case NEON::BI__builtin_neon_vcvtpq_s64_v:
12810 case NEON::BI__builtin_neon_vcvtp_u64_v:
12811 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
12812 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
12814 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtp");
12816 case NEON::BI__builtin_neon_vmulx_v:
12817 case NEON::BI__builtin_neon_vmulxq_v: {
12818 Int = Intrinsic::aarch64_neon_fmulx;
12819 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmulx");
12821 case NEON::BI__builtin_neon_vmulxh_lane_f16:
12822 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
12825 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
12826 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12828 Int = Intrinsic::aarch64_neon_fmulx;
12829 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmulx");
12831 case NEON::BI__builtin_neon_vmul_lane_v:
12832 case NEON::BI__builtin_neon_vmul_laneq_v: {
12835 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
12837 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
12838 llvm::FixedVectorType *VTy =
12840 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
12841 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12842 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
12843 return Builder.CreateBitCast(Result, Ty);
12845 case NEON::BI__builtin_neon_vnegd_s64:
12846 return Builder.CreateNeg(EmitScalarExpr(E->
getArg(0)),
"vnegd");
12847 case NEON::BI__builtin_neon_vnegh_f16:
12848 return Builder.CreateFNeg(EmitScalarExpr(E->
getArg(0)),
"vnegh");
12849 case NEON::BI__builtin_neon_vpmaxnm_v:
12850 case NEON::BI__builtin_neon_vpmaxnmq_v: {
12851 Int = Intrinsic::aarch64_neon_fmaxnmp;
12852 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmaxnm");
12854 case NEON::BI__builtin_neon_vpminnm_v:
12855 case NEON::BI__builtin_neon_vpminnmq_v: {
12856 Int = Intrinsic::aarch64_neon_fminnmp;
12857 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpminnm");
12859 case NEON::BI__builtin_neon_vsqrth_f16: {
12860 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12861 Int = Builder.getIsFPConstrained()
12862 ? Intrinsic::experimental_constrained_sqrt
12864 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vsqrt");
12866 case NEON::BI__builtin_neon_vsqrt_v:
12867 case NEON::BI__builtin_neon_vsqrtq_v: {
12868 Int = Builder.getIsFPConstrained()
12869 ? Intrinsic::experimental_constrained_sqrt
12871 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12872 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vsqrt");
12874 case NEON::BI__builtin_neon_vrbit_v:
12875 case NEON::BI__builtin_neon_vrbitq_v: {
12876 Int = Intrinsic::bitreverse;
12877 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrbit");
12879 case NEON::BI__builtin_neon_vaddv_u8:
12883 case NEON::BI__builtin_neon_vaddv_s8: {
12884 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12886 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
12887 llvm::Type *Tys[2] = { Ty, VTy };
12888 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12889 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
12890 return Builder.CreateTrunc(Ops[0], Int8Ty);
12892 case NEON::BI__builtin_neon_vaddv_u16:
12895 case NEON::BI__builtin_neon_vaddv_s16: {
12896 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12898 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
12899 llvm::Type *Tys[2] = { Ty, VTy };
12900 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12901 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
12902 return Builder.CreateTrunc(Ops[0], Int16Ty);
12904 case NEON::BI__builtin_neon_vaddvq_u8:
12907 case NEON::BI__builtin_neon_vaddvq_s8: {
12908 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12910 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
12911 llvm::Type *Tys[2] = { Ty, VTy };
12912 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12913 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
12914 return Builder.CreateTrunc(Ops[0], Int8Ty);
12916 case NEON::BI__builtin_neon_vaddvq_u16:
12919 case NEON::BI__builtin_neon_vaddvq_s16: {
12920 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12922 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
12923 llvm::Type *Tys[2] = { Ty, VTy };
12924 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12925 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
12926 return Builder.CreateTrunc(Ops[0], Int16Ty);
12928 case NEON::BI__builtin_neon_vmaxv_u8: {
12929 Int = Intrinsic::aarch64_neon_umaxv;
12931 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
12932 llvm::Type *Tys[2] = { Ty, VTy };
12933 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12934 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
12935 return Builder.CreateTrunc(Ops[0], Int8Ty);
12937 case NEON::BI__builtin_neon_vmaxv_u16: {
12938 Int = Intrinsic::aarch64_neon_umaxv;
12940 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
12941 llvm::Type *Tys[2] = { Ty, VTy };
12942 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12943 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
12944 return Builder.CreateTrunc(Ops[0], Int16Ty);
12946 case NEON::BI__builtin_neon_vmaxvq_u8: {
12947 Int = Intrinsic::aarch64_neon_umaxv;
12949 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
12950 llvm::Type *Tys[2] = { Ty, VTy };
12951 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12952 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
12953 return Builder.CreateTrunc(Ops[0], Int8Ty);
12955 case NEON::BI__builtin_neon_vmaxvq_u16: {
12956 Int = Intrinsic::aarch64_neon_umaxv;
12958 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
12959 llvm::Type *Tys[2] = { Ty, VTy };
12960 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12961 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
12962 return Builder.CreateTrunc(Ops[0], Int16Ty);
12964 case NEON::BI__builtin_neon_vmaxv_s8: {
12965 Int = Intrinsic::aarch64_neon_smaxv;
12967 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
12968 llvm::Type *Tys[2] = { Ty, VTy };
12969 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12970 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
12971 return Builder.CreateTrunc(Ops[0], Int8Ty);
12973 case NEON::BI__builtin_neon_vmaxv_s16: {
12974 Int = Intrinsic::aarch64_neon_smaxv;
12976 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
12977 llvm::Type *Tys[2] = { Ty, VTy };
12978 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12979 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
12980 return Builder.CreateTrunc(Ops[0], Int16Ty);
12982 case NEON::BI__builtin_neon_vmaxvq_s8: {
12983 Int = Intrinsic::aarch64_neon_smaxv;
12985 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
12986 llvm::Type *Tys[2] = { Ty, VTy };
12987 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12988 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
12989 return Builder.CreateTrunc(Ops[0], Int8Ty);
12991 case NEON::BI__builtin_neon_vmaxvq_s16: {
12992 Int = Intrinsic::aarch64_neon_smaxv;
12994 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
12995 llvm::Type *Tys[2] = { Ty, VTy };
12996 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
12997 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
12998 return Builder.CreateTrunc(Ops[0], Int16Ty);
13000 case NEON::BI__builtin_neon_vmaxv_f16: {
13001 Int = Intrinsic::aarch64_neon_fmaxv;
13003 VTy = llvm::FixedVectorType::get(HalfTy, 4);
13004 llvm::Type *Tys[2] = { Ty, VTy };
13005 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13006 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
13007 return Builder.CreateTrunc(Ops[0], HalfTy);
13009 case NEON::BI__builtin_neon_vmaxvq_f16: {
13010 Int = Intrinsic::aarch64_neon_fmaxv;
13012 VTy = llvm::FixedVectorType::get(HalfTy, 8);
13013 llvm::Type *Tys[2] = { Ty, VTy };
13014 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13015 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
13016 return Builder.CreateTrunc(Ops[0], HalfTy);
13018 case NEON::BI__builtin_neon_vminv_u8: {
13019 Int = Intrinsic::aarch64_neon_uminv;
13021 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
13022 llvm::Type *Tys[2] = { Ty, VTy };
13023 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13024 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13025 return Builder.CreateTrunc(Ops[0], Int8Ty);
13027 case NEON::BI__builtin_neon_vminv_u16: {
13028 Int = Intrinsic::aarch64_neon_uminv;
13030 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
13031 llvm::Type *Tys[2] = { Ty, VTy };
13032 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13033 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13034 return Builder.CreateTrunc(Ops[0], Int16Ty);
13036 case NEON::BI__builtin_neon_vminvq_u8: {
13037 Int = Intrinsic::aarch64_neon_uminv;
13039 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
13040 llvm::Type *Tys[2] = { Ty, VTy };
13041 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13042 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13043 return Builder.CreateTrunc(Ops[0], Int8Ty);
13045 case NEON::BI__builtin_neon_vminvq_u16: {
13046 Int = Intrinsic::aarch64_neon_uminv;
13048 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
13049 llvm::Type *Tys[2] = { Ty, VTy };
13050 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13051 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13052 return Builder.CreateTrunc(Ops[0], Int16Ty);
13054 case NEON::BI__builtin_neon_vminv_s8: {
13055 Int = Intrinsic::aarch64_neon_sminv;
13057 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
13058 llvm::Type *Tys[2] = { Ty, VTy };
13059 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13060 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13061 return Builder.CreateTrunc(Ops[0], Int8Ty);
13063 case NEON::BI__builtin_neon_vminv_s16: {
13064 Int = Intrinsic::aarch64_neon_sminv;
13066 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
13067 llvm::Type *Tys[2] = { Ty, VTy };
13068 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13069 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13070 return Builder.CreateTrunc(Ops[0], Int16Ty);
13072 case NEON::BI__builtin_neon_vminvq_s8: {
13073 Int = Intrinsic::aarch64_neon_sminv;
13075 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
13076 llvm::Type *Tys[2] = { Ty, VTy };
13077 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13078 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13079 return Builder.CreateTrunc(Ops[0], Int8Ty);
13081 case NEON::BI__builtin_neon_vminvq_s16: {
13082 Int = Intrinsic::aarch64_neon_sminv;
13084 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
13085 llvm::Type *Tys[2] = { Ty, VTy };
13086 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13087 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13088 return Builder.CreateTrunc(Ops[0], Int16Ty);
13090 case NEON::BI__builtin_neon_vminv_f16: {
13091 Int = Intrinsic::aarch64_neon_fminv;
13093 VTy = llvm::FixedVectorType::get(HalfTy, 4);
13094 llvm::Type *Tys[2] = { Ty, VTy };
13095 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13096 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13097 return Builder.CreateTrunc(Ops[0], HalfTy);
13099 case NEON::BI__builtin_neon_vminvq_f16: {
13100 Int = Intrinsic::aarch64_neon_fminv;
13102 VTy = llvm::FixedVectorType::get(HalfTy, 8);
13103 llvm::Type *Tys[2] = { Ty, VTy };
13104 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13105 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
13106 return Builder.CreateTrunc(Ops[0], HalfTy);
13108 case NEON::BI__builtin_neon_vmaxnmv_f16: {
13109 Int = Intrinsic::aarch64_neon_fmaxnmv;
13111 VTy = llvm::FixedVectorType::get(HalfTy, 4);
13112 llvm::Type *Tys[2] = { Ty, VTy };
13113 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13114 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxnmv");
13115 return Builder.CreateTrunc(Ops[0], HalfTy);
13117 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
13118 Int = Intrinsic::aarch64_neon_fmaxnmv;
13120 VTy = llvm::FixedVectorType::get(HalfTy, 8);
13121 llvm::Type *Tys[2] = { Ty, VTy };
13122 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13123 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxnmv");
13124 return Builder.CreateTrunc(Ops[0], HalfTy);
13126 case NEON::BI__builtin_neon_vminnmv_f16: {
13127 Int = Intrinsic::aarch64_neon_fminnmv;
13129 VTy = llvm::FixedVectorType::get(HalfTy, 4);
13130 llvm::Type *Tys[2] = { Ty, VTy };
13131 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13132 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminnmv");
13133 return Builder.CreateTrunc(Ops[0], HalfTy);
13135 case NEON::BI__builtin_neon_vminnmvq_f16: {
13136 Int = Intrinsic::aarch64_neon_fminnmv;
13138 VTy = llvm::FixedVectorType::get(HalfTy, 8);
13139 llvm::Type *Tys[2] = { Ty, VTy };
13140 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13141 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminnmv");
13142 return Builder.CreateTrunc(Ops[0], HalfTy);
13144 case NEON::BI__builtin_neon_vmul_n_f64: {
13145 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
13146 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(1)), DoubleTy);
13147 return Builder.CreateFMul(Ops[0], RHS);
13149 case NEON::BI__builtin_neon_vaddlv_u8: {
13150 Int = Intrinsic::aarch64_neon_uaddlv;
13152 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
13153 llvm::Type *Tys[2] = { Ty, VTy };
13154 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13155 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
13156 return Builder.CreateTrunc(Ops[0], Int16Ty);
13158 case NEON::BI__builtin_neon_vaddlv_u16: {
13159 Int = Intrinsic::aarch64_neon_uaddlv;
13161 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
13162 llvm::Type *Tys[2] = { Ty, VTy };
13163 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13164 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
13166 case NEON::BI__builtin_neon_vaddlvq_u8: {
13167 Int = Intrinsic::aarch64_neon_uaddlv;
13169 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
13170 llvm::Type *Tys[2] = { Ty, VTy };
13171 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13172 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
13173 return Builder.CreateTrunc(Ops[0], Int16Ty);
13175 case NEON::BI__builtin_neon_vaddlvq_u16: {
13176 Int = Intrinsic::aarch64_neon_uaddlv;
13178 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
13179 llvm::Type *Tys[2] = { Ty, VTy };
13180 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13181 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
13183 case NEON::BI__builtin_neon_vaddlv_s8: {
13184 Int = Intrinsic::aarch64_neon_saddlv;
13186 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
13187 llvm::Type *Tys[2] = { Ty, VTy };
13188 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13189 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
13190 return Builder.CreateTrunc(Ops[0], Int16Ty);
13192 case NEON::BI__builtin_neon_vaddlv_s16: {
13193 Int = Intrinsic::aarch64_neon_saddlv;
13195 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
13196 llvm::Type *Tys[2] = { Ty, VTy };
13197 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13198 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
13200 case NEON::BI__builtin_neon_vaddlvq_s8: {
13201 Int = Intrinsic::aarch64_neon_saddlv;
13203 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
13204 llvm::Type *Tys[2] = { Ty, VTy };
13205 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13206 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
13207 return Builder.CreateTrunc(Ops[0], Int16Ty);
13209 case NEON::BI__builtin_neon_vaddlvq_s16: {
13210 Int = Intrinsic::aarch64_neon_saddlv;
13212 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
13213 llvm::Type *Tys[2] = { Ty, VTy };
13214 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
13215 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
13217 case NEON::BI__builtin_neon_vsri_n_v:
13218 case NEON::BI__builtin_neon_vsriq_n_v: {
13219 Int = Intrinsic::aarch64_neon_vsri;
13220 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
13221 return EmitNeonCall(Intrin, Ops,
"vsri_n");
13223 case NEON::BI__builtin_neon_vsli_n_v:
13224 case NEON::BI__builtin_neon_vsliq_n_v: {
13225 Int = Intrinsic::aarch64_neon_vsli;
13226 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
13227 return EmitNeonCall(Intrin, Ops,
"vsli_n");
13229 case NEON::BI__builtin_neon_vsra_n_v:
13230 case NEON::BI__builtin_neon_vsraq_n_v:
13231 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
13232 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn,
"vsra_n");
13233 return Builder.CreateAdd(Ops[0], Ops[1]);
13234 case NEON::BI__builtin_neon_vrsra_n_v:
13235 case NEON::BI__builtin_neon_vrsraq_n_v: {
13236 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13238 TmpOps.push_back(Ops[1]);
13239 TmpOps.push_back(Ops[2]);
13240 Function* F = CGM.getIntrinsic(Int, Ty);
13241 llvm::Value *tmp = EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
13242 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
13243 return Builder.CreateAdd(Ops[0], tmp);
13245 case NEON::BI__builtin_neon_vld1_v:
13246 case NEON::BI__builtin_neon_vld1q_v: {
13247 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.
getAlignment());
13249 case NEON::BI__builtin_neon_vst1_v:
13250 case NEON::BI__builtin_neon_vst1q_v:
13251 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
13252 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.
getAlignment());
13253 case NEON::BI__builtin_neon_vld1_lane_v:
13254 case NEON::BI__builtin_neon_vld1q_lane_v: {
13255 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13256 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
13258 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
13260 case NEON::BI__builtin_neon_vldap1_lane_s64:
13261 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
13262 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13263 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
13264 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
13265 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13267 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
13269 case NEON::BI__builtin_neon_vld1_dup_v:
13270 case NEON::BI__builtin_neon_vld1q_dup_v: {
13271 Value *
V = PoisonValue::get(Ty);
13272 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
13274 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
13275 Ops[0] = Builder.CreateInsertElement(
V, Ops[0], CI);
13276 return EmitNeonSplat(Ops[0], CI);
13278 case NEON::BI__builtin_neon_vst1_lane_v:
13279 case NEON::BI__builtin_neon_vst1q_lane_v:
13280 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13281 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
13282 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.
getAlignment());
13283 case NEON::BI__builtin_neon_vstl1_lane_s64:
13284 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
13285 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13286 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
13287 llvm::StoreInst *SI =
13288 Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.
getAlignment());
13289 SI->setAtomic(llvm::AtomicOrdering::Release);
13292 case NEON::BI__builtin_neon_vld2_v:
13293 case NEON::BI__builtin_neon_vld2q_v: {
13294 llvm::Type *Tys[2] = {VTy, UnqualPtrTy};
13295 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
13296 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
13297 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13299 case NEON::BI__builtin_neon_vld3_v:
13300 case NEON::BI__builtin_neon_vld3q_v: {
13301 llvm::Type *Tys[2] = {VTy, UnqualPtrTy};
13302 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
13303 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
13304 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13306 case NEON::BI__builtin_neon_vld4_v:
13307 case NEON::BI__builtin_neon_vld4q_v: {
13308 llvm::Type *Tys[2] = {VTy, UnqualPtrTy};
13309 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
13310 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
13311 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13313 case NEON::BI__builtin_neon_vld2_dup_v:
13314 case NEON::BI__builtin_neon_vld2q_dup_v: {
13315 llvm::Type *Tys[2] = {VTy, UnqualPtrTy};
13316 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
13317 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
13318 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13320 case NEON::BI__builtin_neon_vld3_dup_v:
13321 case NEON::BI__builtin_neon_vld3q_dup_v: {
13322 llvm::Type *Tys[2] = {VTy, UnqualPtrTy};
13323 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
13324 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
13325 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13327 case NEON::BI__builtin_neon_vld4_dup_v:
13328 case NEON::BI__builtin_neon_vld4q_dup_v: {
13329 llvm::Type *Tys[2] = {VTy, UnqualPtrTy};
13330 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
13331 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
13332 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13334 case NEON::BI__builtin_neon_vld2_lane_v:
13335 case NEON::BI__builtin_neon_vld2q_lane_v: {
13336 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13337 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
13338 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13339 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13340 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13341 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
13342 Ops[1] = Builder.CreateCall(F,
ArrayRef(Ops).slice(1),
"vld2_lane");
13343 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13345 case NEON::BI__builtin_neon_vld3_lane_v:
13346 case NEON::BI__builtin_neon_vld3q_lane_v: {
13347 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13348 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
13349 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13350 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13351 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13352 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
13353 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
13354 Ops[1] = Builder.CreateCall(F,
ArrayRef(Ops).slice(1),
"vld3_lane");
13355 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13357 case NEON::BI__builtin_neon_vld4_lane_v:
13358 case NEON::BI__builtin_neon_vld4q_lane_v: {
13359 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13360 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
13361 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13362 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13363 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13364 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
13365 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
13366 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
13367 Ops[1] = Builder.CreateCall(F,
ArrayRef(Ops).slice(1),
"vld4_lane");
13368 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13370 case NEON::BI__builtin_neon_vst2_v:
13371 case NEON::BI__builtin_neon_vst2q_v: {
13372 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13373 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13374 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
13377 case NEON::BI__builtin_neon_vst2_lane_v:
13378 case NEON::BI__builtin_neon_vst2q_lane_v: {
13379 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13380 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
13381 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13382 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
13385 case NEON::BI__builtin_neon_vst3_v:
13386 case NEON::BI__builtin_neon_vst3q_v: {
13387 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13388 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13389 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
13392 case NEON::BI__builtin_neon_vst3_lane_v:
13393 case NEON::BI__builtin_neon_vst3q_lane_v: {
13394 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13395 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
13396 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13397 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
13400 case NEON::BI__builtin_neon_vst4_v:
13401 case NEON::BI__builtin_neon_vst4q_v: {
13402 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13403 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13404 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
13407 case NEON::BI__builtin_neon_vst4_lane_v:
13408 case NEON::BI__builtin_neon_vst4q_lane_v: {
13409 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13410 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
13411 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13412 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
13415 case NEON::BI__builtin_neon_vtrn_v:
13416 case NEON::BI__builtin_neon_vtrnq_v: {
13417 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13418 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13419 Value *SV =
nullptr;
13421 for (
unsigned vi = 0; vi != 2; ++vi) {
13423 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13424 Indices.push_back(i+vi);
13425 Indices.push_back(i+e+vi);
13427 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13428 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
13429 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
13433 case NEON::BI__builtin_neon_vuzp_v:
13434 case NEON::BI__builtin_neon_vuzpq_v: {
13435 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13436 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13437 Value *SV =
nullptr;
13439 for (
unsigned vi = 0; vi != 2; ++vi) {
13441 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13442 Indices.push_back(2*i+vi);
13444 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13445 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
13446 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
13450 case NEON::BI__builtin_neon_vzip_v:
13451 case NEON::BI__builtin_neon_vzipq_v: {
13452 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13453 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13454 Value *SV =
nullptr;
13456 for (
unsigned vi = 0; vi != 2; ++vi) {
13458 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13459 Indices.push_back((i + vi*e) >> 1);
13460 Indices.push_back(((i + vi*e) >> 1)+e);
13462 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13463 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
13464 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
13468 case NEON::BI__builtin_neon_vqtbl1q_v: {
13469 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
13472 case NEON::BI__builtin_neon_vqtbl2q_v: {
13473 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
13476 case NEON::BI__builtin_neon_vqtbl3q_v: {
13477 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
13480 case NEON::BI__builtin_neon_vqtbl4q_v: {
13481 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
13484 case NEON::BI__builtin_neon_vqtbx1q_v: {
13485 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
13488 case NEON::BI__builtin_neon_vqtbx2q_v: {
13489 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
13492 case NEON::BI__builtin_neon_vqtbx3q_v: {
13493 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
13496 case NEON::BI__builtin_neon_vqtbx4q_v: {
13497 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
13500 case NEON::BI__builtin_neon_vsqadd_v:
13501 case NEON::BI__builtin_neon_vsqaddq_v: {
13502 Int = Intrinsic::aarch64_neon_usqadd;
13503 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vsqadd");
13505 case NEON::BI__builtin_neon_vuqadd_v:
13506 case NEON::BI__builtin_neon_vuqaddq_v: {
13507 Int = Intrinsic::aarch64_neon_suqadd;
13508 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vuqadd");
13515 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
13516 BuiltinID == BPF::BI__builtin_btf_type_id ||
13517 BuiltinID == BPF::BI__builtin_preserve_type_info ||
13518 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
13519 "unexpected BPF builtin");
13524 static uint32_t BuiltinSeqNum;
13526 switch (BuiltinID) {
13528 llvm_unreachable(
"Unexpected BPF builtin");
13529 case BPF::BI__builtin_preserve_field_info: {
13533 if (!getDebugInfo()) {
13535 "using __builtin_preserve_field_info() without -g");
13536 return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*
this)
13537 : EmitLValue(Arg).emitRawPointer(*
this);
13541 bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
13542 IsInPreservedAIRegion =
true;
13543 Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*
this)
13544 : EmitLValue(Arg).emitRawPointer(*
this);
13545 IsInPreservedAIRegion = OldIsInPreservedAIRegion;
13547 ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
13548 Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue());
13551 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
13552 &CGM.getModule(), llvm::Intrinsic::bpf_preserve_field_info,
13553 {FieldAddr->getType()});
13554 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
13556 case BPF::BI__builtin_btf_type_id:
13557 case BPF::BI__builtin_preserve_type_info: {
13558 if (!getDebugInfo()) {
13559 CGM.Error(E->
getExprLoc(),
"using builtin function without -g");
13564 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
13567 ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
13568 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
13569 Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
13571 llvm::Function *FnDecl;
13572 if (BuiltinID == BPF::BI__builtin_btf_type_id)
13573 FnDecl = llvm::Intrinsic::getDeclaration(
13574 &CGM.getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
13576 FnDecl = llvm::Intrinsic::getDeclaration(
13577 &CGM.getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
13578 CallInst *Fn = Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
13579 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13582 case BPF::BI__builtin_preserve_enum_value: {
13583 if (!getDebugInfo()) {
13584 CGM.Error(E->
getExprLoc(),
"using builtin function without -g");
13589 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
13593 const auto *UO = cast<UnaryOperator>(Arg0->
IgnoreParens());
13594 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
13595 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
13596 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
13598 auto InitVal = Enumerator->getInitVal();
13599 std::string InitValStr;
13600 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
13601 InitValStr = std::to_string(InitVal.getSExtValue());
13603 InitValStr = std::to_string(InitVal.getZExtValue());
13604 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
13605 Value *EnumStrVal = Builder.CreateGlobalStringPtr(EnumStr);
13607 ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
13608 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
13609 Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
13611 llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
13612 &CGM.getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
13614 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
13615 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13623 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
13624 "Not a power-of-two sized vector!");
13625 bool AllConstants =
true;
13626 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
13627 AllConstants &= isa<Constant>(Ops[i]);
13630 if (AllConstants) {
13632 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13633 CstOps.push_back(cast<Constant>(Ops[i]));
13634 return llvm::ConstantVector::get(CstOps);
13638 Value *Result = llvm::PoisonValue::get(
13639 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
13641 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13642 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt64(i));
13649 unsigned NumElts) {
13651 auto *MaskTy = llvm::FixedVectorType::get(
13653 cast<IntegerType>(Mask->
getType())->getBitWidth());
13654 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13660 for (
unsigned i = 0; i != NumElts; ++i)
13662 MaskVec = CGF.
Builder.CreateShuffleVector(
13663 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
13671 Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType()));
13675 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
13677 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
13682 llvm::Type *Ty = Ops[1]->getType();
13684 Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType()));
13687 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
13689 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
13694 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
13695 Value *Ptr = Ops[0];
13698 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
13700 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
13702 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
13708 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13712 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
13713 : Intrinsic::x86_avx512_mask_expand;
13715 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
13720 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13721 Value *Ptr = Ops[0];
13725 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
13727 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
13732 bool InvertLHS =
false) {
13733 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13738 LHS = CGF.
Builder.CreateNot(LHS);
13740 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
13741 Ops[0]->getType());
13745 Value *Amt,
bool IsRight) {
13746 llvm::Type *Ty = Op0->
getType();
13752 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
13753 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
13754 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
13757 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
13759 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
13764 Value *Op0 = Ops[0];
13765 Value *Op1 = Ops[1];
13766 llvm::Type *Ty = Op0->
getType();
13767 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
13769 CmpInst::Predicate Pred;
13772 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13775 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
13778 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13781 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13784 Pred = ICmpInst::ICMP_EQ;
13787 Pred = ICmpInst::ICMP_NE;
13790 return llvm::Constant::getNullValue(Ty);
13792 return llvm::Constant::getAllOnesValue(Ty);
13794 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
13806 if (
const auto *C = dyn_cast<Constant>(Mask))
13807 if (C->isAllOnesValue())
13811 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
13813 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13819 if (
const auto *C = dyn_cast<Constant>(Mask))
13820 if (C->isAllOnesValue())
13823 auto *MaskTy = llvm::FixedVectorType::get(
13824 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
13825 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13827 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13831 unsigned NumElts,
Value *MaskIn) {
13833 const auto *C = dyn_cast<Constant>(MaskIn);
13834 if (!C || !C->isAllOnesValue())
13840 for (
unsigned i = 0; i != NumElts; ++i)
13842 for (
unsigned i = NumElts; i != 8; ++i)
13843 Indices[i] = i % NumElts + NumElts;
13844 Cmp = CGF.
Builder.CreateShuffleVector(
13845 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
13848 return CGF.
Builder.CreateBitCast(Cmp,
13855 assert((Ops.size() == 2 || Ops.size() == 4) &&
13856 "Unexpected number of arguments");
13858 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
13862 Cmp = Constant::getNullValue(
13863 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13864 }
else if (CC == 7) {
13865 Cmp = Constant::getAllOnesValue(
13866 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13868 ICmpInst::Predicate Pred;
13870 default: llvm_unreachable(
"Unknown condition code");
13871 case 0: Pred = ICmpInst::ICMP_EQ;
break;
13872 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
13873 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
13874 case 4: Pred = ICmpInst::ICMP_NE;
break;
13875 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
13876 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
13878 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
13881 Value *MaskIn =
nullptr;
13882 if (Ops.size() == 4)
13889 Value *Zero = Constant::getNullValue(In->getType());
13895 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
13896 llvm::Type *Ty = Ops[1]->getType();
13900 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
13901 : Intrinsic::x86_avx512_uitofp_round;
13903 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
13906 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
13907 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
13918 bool Subtract =
false;
13920 switch (BuiltinID) {
13922 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13925 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13926 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13927 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13928 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
13930 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13933 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13934 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13935 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13936 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
13938 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13941 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13942 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13943 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13944 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
13945 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13948 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13949 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13950 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13951 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
13952 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13955 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13956 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13957 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13958 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
13960 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13963 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13964 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13965 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13966 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
13975 C = CGF.
Builder.CreateFNeg(C);
13980 if (IID != Intrinsic::not_intrinsic &&
13981 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (
uint64_t)4 ||
13984 Res = CGF.
Builder.CreateCall(Intr, {A, B, C, Ops.back() });
13986 llvm::Type *Ty = A->
getType();
13988 if (CGF.
Builder.getIsFPConstrained()) {
13990 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
13991 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B, C});
13994 Res = CGF.
Builder.CreateCall(FMA, {A, B, C});
13999 Value *MaskFalseVal =
nullptr;
14000 switch (BuiltinID) {
14001 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
14002 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
14003 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
14004 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
14005 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
14006 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14007 MaskFalseVal = Ops[0];
14009 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
14010 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
14011 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
14012 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14013 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14014 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14015 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
14017 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
14018 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
14019 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
14020 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
14021 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
14022 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
14023 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14024 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14025 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14026 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14027 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14028 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14029 MaskFalseVal = Ops[2];
14041 bool ZeroMask =
false,
unsigned PTIdx = 0,
14042 bool NegAcc =
false) {
14044 if (Ops.size() > 4)
14045 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
14048 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
14057 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
14059 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
14062 IID = Intrinsic::x86_avx512_vfmadd_f32;
14065 IID = Intrinsic::x86_avx512_vfmadd_f64;
14068 llvm_unreachable(
"Unexpected size");
14071 {Ops[0], Ops[1], Ops[2], Ops[4]});
14072 }
else if (CGF.
Builder.getIsFPConstrained()) {
14075 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
14076 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
14079 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
14082 if (Ops.size() > 3) {
14083 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
14089 if (NegAcc && PTIdx == 2)
14099 llvm::Type *Ty = Ops[0]->getType();
14101 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
14102 Ty->getPrimitiveSizeInBits() / 64);
14108 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
14109 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
14110 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
14111 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
14112 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
14115 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
14116 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
14117 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
14120 return CGF.
Builder.CreateMul(LHS, RHS);
14128 llvm::Type *Ty = Ops[0]->getType();
14130 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
14131 unsigned EltWidth = Ty->getScalarSizeInBits();
14133 if (VecWidth == 128 && EltWidth == 32)
14134 IID = Intrinsic::x86_avx512_pternlog_d_128;
14135 else if (VecWidth == 256 && EltWidth == 32)
14136 IID = Intrinsic::x86_avx512_pternlog_d_256;
14137 else if (VecWidth == 512 && EltWidth == 32)
14138 IID = Intrinsic::x86_avx512_pternlog_d_512;
14139 else if (VecWidth == 128 && EltWidth == 64)
14140 IID = Intrinsic::x86_avx512_pternlog_q_128;
14141 else if (VecWidth == 256 && EltWidth == 64)
14142 IID = Intrinsic::x86_avx512_pternlog_q_256;
14143 else if (VecWidth == 512 && EltWidth == 64)
14144 IID = Intrinsic::x86_avx512_pternlog_q_512;
14146 llvm_unreachable(
"Unexpected intrinsic");
14150 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
14155 llvm::Type *DstTy) {
14156 unsigned NumberOfElements =
14157 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14159 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
14164 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
14165 return EmitX86CpuIs(CPUStr);
14171 llvm::Type *DstTy) {
14172 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
14173 "Unknown cvtph2ps intrinsic");
14176 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
14179 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
14182 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14183 Value *Src = Ops[0];
14187 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
14188 assert(NumDstElts == 4 &&
"Unexpected vector size");
14193 auto *HalfTy = llvm::FixedVectorType::get(
14195 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
14198 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
14200 if (Ops.size() >= 3)
14205 Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
14207 llvm::Type *Int32Ty = Builder.getInt32Ty();
14215 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
14216 llvm::ArrayType::get(Int32Ty, 1));
14219 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy,
"__cpu_model");
14220 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14226 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
14228 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
14230 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14232 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14234 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14236 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14237 #include
"llvm/TargetParser/X86TargetParser.def"
14239 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
14242 llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0),
14243 ConstantInt::get(Int32Ty, Index)};
14244 llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
14245 CpuValue = Builder.CreateAlignedLoad(Int32Ty, CpuValue,
14249 return Builder.CreateICmpEQ(CpuValue,
14250 llvm::ConstantInt::get(Int32Ty,
Value));
14253 Value *CodeGenFunction::EmitX86CpuSupports(
const CallExpr *E) {
14255 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14256 if (!getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14257 return Builder.getFalse();
14258 return EmitX86CpuSupports(FeatureStr);
14262 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
14266 CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) {
14267 Value *Result = Builder.getTrue();
14268 if (FeatureMask[0] != 0) {
14275 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
14276 llvm::ArrayType::get(Int32Ty, 1));
14279 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy,
"__cpu_model");
14280 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14284 Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3),
14285 Builder.getInt32(0)};
14286 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
14287 Value *Features = Builder.CreateAlignedLoad(Int32Ty, CpuFeatures,
14291 Value *Mask = Builder.getInt32(FeatureMask[0]);
14292 Value *Bitset = Builder.CreateAnd(Features, Mask);
14293 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
14294 Result = Builder.CreateAnd(Result, Cmp);
14297 llvm::Type *ATy = llvm::ArrayType::get(Int32Ty, 3);
14298 llvm::Constant *CpuFeatures2 =
14299 CGM.CreateRuntimeVariable(ATy,
"__cpu_features2");
14300 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
14301 for (
int i = 1; i != 4; ++i) {
14302 const uint32_t M = FeatureMask[i];
14305 Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(i - 1)};
14306 Value *Features = Builder.CreateAlignedLoad(
14307 Int32Ty, Builder.CreateGEP(ATy, CpuFeatures2, Idxs),
14310 Value *Mask = Builder.getInt32(M);
14311 Value *Bitset = Builder.CreateAnd(Features, Mask);
14312 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
14313 Result = Builder.CreateAnd(Result, Cmp);
14319 Value *CodeGenFunction::EmitAArch64CpuInit() {
14320 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
false);
14321 llvm::FunctionCallee
Func =
14322 CGM.CreateRuntimeFunction(FTy,
"__init_cpu_features_resolver");
14323 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14324 cast<llvm::GlobalValue>(
Func.getCallee())
14325 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14326 return Builder.CreateCall(Func);
14329 Value *CodeGenFunction::EmitX86CpuInit() {
14330 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
14332 llvm::FunctionCallee
Func =
14333 CGM.CreateRuntimeFunction(FTy,
"__cpu_indicator_init");
14334 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14335 cast<llvm::GlobalValue>(
Func.getCallee())
14336 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14337 return Builder.CreateCall(Func);
14340 Value *CodeGenFunction::EmitAArch64CpuSupports(
const CallExpr *E) {
14342 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
14344 ArgStr.split(Features,
"+");
14345 for (
auto &Feature : Features) {
14346 Feature = Feature.trim();
14347 if (!llvm::AArch64::parseArchExtension(Feature))
14348 return Builder.getFalse();
14349 if (Feature !=
"default")
14350 Features.push_back(Feature);
14352 return EmitAArch64CpuSupports(Features);
14357 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
14358 Value *Result = Builder.getTrue();
14359 if (FeaturesMask != 0) {
14364 llvm::Type *STy = llvm::StructType::get(Int64Ty);
14365 llvm::Constant *AArch64CPUFeatures =
14366 CGM.CreateRuntimeVariable(STy,
"__aarch64_cpu_features");
14367 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(
true);
14368 llvm::Value *CpuFeatures = Builder.CreateGEP(
14369 STy, AArch64CPUFeatures,
14370 {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 0)});
14371 Value *Features = Builder.CreateAlignedLoad(Int64Ty, CpuFeatures,
14373 Value *Mask = Builder.getInt64(FeaturesMask);
14374 Value *Bitset = Builder.CreateAnd(Features, Mask);
14375 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
14376 Result = Builder.CreateAnd(Result, Cmp);
14383 if (BuiltinID == Builtin::BI__builtin_cpu_is)
14384 return EmitX86CpuIs(E);
14385 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
14386 return EmitX86CpuSupports(E);
14387 if (BuiltinID == Builtin::BI__builtin_cpu_init)
14388 return EmitX86CpuInit();
14393 return EmitMSVCBuiltinExpr(*MsvcIntId, E);
14396 bool IsMaskFCmp =
false;
14397 bool IsConjFMA =
false;
14400 unsigned ICEArguments = 0;
14402 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
14405 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
14406 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E));
14415 auto getCmpIntrinsicCall = [
this, &Ops](
Intrinsic::ID ID,
unsigned Imm) {
14416 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm));
14417 llvm::Function *F = CGM.getIntrinsic(
ID);
14418 return Builder.CreateCall(F, Ops);
14426 auto getVectorFCmpIR = [
this, &Ops, E](CmpInst::Predicate Pred,
14427 bool IsSignaling) {
14431 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
14433 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
14434 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
14435 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
14436 Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
14437 return Builder.CreateBitCast(Sext, FPVecTy);
14440 switch (BuiltinID) {
14441 default:
return nullptr;
14442 case X86::BI_mm_prefetch: {
14444 ConstantInt *C = cast<ConstantInt>(Ops[1]);
14445 Value *RW = ConstantInt::get(Int32Ty, (C->getZExtValue() >> 2) & 0x1);
14446 Value *Locality = ConstantInt::get(Int32Ty, C->getZExtValue() & 0x3);
14447 Value *
Data = ConstantInt::get(Int32Ty, 1);
14449 return Builder.CreateCall(F, {
Address, RW, Locality,
Data});
14451 case X86::BI_mm_clflush: {
14452 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
14455 case X86::BI_mm_lfence: {
14456 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
14458 case X86::BI_mm_mfence: {
14459 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
14461 case X86::BI_mm_sfence: {
14462 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
14464 case X86::BI_mm_pause: {
14465 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
14467 case X86::BI__rdtsc: {
14468 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
14470 case X86::BI__builtin_ia32_rdtscp: {
14471 Value *Call = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtscp));
14472 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
14474 return Builder.CreateExtractValue(Call, 0);
14476 case X86::BI__builtin_ia32_lzcnt_u16:
14477 case X86::BI__builtin_ia32_lzcnt_u32:
14478 case X86::BI__builtin_ia32_lzcnt_u64: {
14479 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
14480 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
14482 case X86::BI__builtin_ia32_tzcnt_u16:
14483 case X86::BI__builtin_ia32_tzcnt_u32:
14484 case X86::BI__builtin_ia32_tzcnt_u64: {
14485 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
14486 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
14488 case X86::BI__builtin_ia32_undef128:
14489 case X86::BI__builtin_ia32_undef256:
14490 case X86::BI__builtin_ia32_undef512:
14496 return llvm::Constant::getNullValue(ConvertType(E->
getType()));
14497 case X86::BI__builtin_ia32_vec_init_v8qi:
14498 case X86::BI__builtin_ia32_vec_init_v4hi:
14499 case X86::BI__builtin_ia32_vec_init_v2si:
14500 return Builder.CreateBitCast(BuildVector(Ops),
14501 llvm::Type::getX86_MMXTy(getLLVMContext()));
14502 case X86::BI__builtin_ia32_vec_ext_v2si:
14503 case X86::BI__builtin_ia32_vec_ext_v16qi:
14504 case X86::BI__builtin_ia32_vec_ext_v8hi:
14505 case X86::BI__builtin_ia32_vec_ext_v4si:
14506 case X86::BI__builtin_ia32_vec_ext_v4sf:
14507 case X86::BI__builtin_ia32_vec_ext_v2di:
14508 case X86::BI__builtin_ia32_vec_ext_v32qi:
14509 case X86::BI__builtin_ia32_vec_ext_v16hi:
14510 case X86::BI__builtin_ia32_vec_ext_v8si:
14511 case X86::BI__builtin_ia32_vec_ext_v4di: {
14513 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14514 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
14515 Index &= NumElts - 1;
14518 return Builder.CreateExtractElement(Ops[0], Index);
14520 case X86::BI__builtin_ia32_vec_set_v16qi:
14521 case X86::BI__builtin_ia32_vec_set_v8hi:
14522 case X86::BI__builtin_ia32_vec_set_v4si:
14523 case X86::BI__builtin_ia32_vec_set_v2di:
14524 case X86::BI__builtin_ia32_vec_set_v32qi:
14525 case X86::BI__builtin_ia32_vec_set_v16hi:
14526 case X86::BI__builtin_ia32_vec_set_v8si:
14527 case X86::BI__builtin_ia32_vec_set_v4di: {
14529 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14530 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
14531 Index &= NumElts - 1;
14534 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
14536 case X86::BI_mm_setcsr:
14537 case X86::BI__builtin_ia32_ldmxcsr: {
14539 Builder.CreateStore(Ops[0], Tmp);
14540 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
14543 case X86::BI_mm_getcsr:
14544 case X86::BI__builtin_ia32_stmxcsr: {
14546 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
14548 return Builder.CreateLoad(Tmp,
"stmxcsr");
14550 case X86::BI__builtin_ia32_xsave:
14551 case X86::BI__builtin_ia32_xsave64:
14552 case X86::BI__builtin_ia32_xrstor:
14553 case X86::BI__builtin_ia32_xrstor64:
14554 case X86::BI__builtin_ia32_xsaveopt:
14555 case X86::BI__builtin_ia32_xsaveopt64:
14556 case X86::BI__builtin_ia32_xrstors:
14557 case X86::BI__builtin_ia32_xrstors64:
14558 case X86::BI__builtin_ia32_xsavec:
14559 case X86::BI__builtin_ia32_xsavec64:
14560 case X86::BI__builtin_ia32_xsaves:
14561 case X86::BI__builtin_ia32_xsaves64:
14562 case X86::BI__builtin_ia32_xsetbv:
14563 case X86::BI_xsetbv: {
14565 #define INTRINSIC_X86_XSAVE_ID(NAME) \
14566 case X86::BI__builtin_ia32_##NAME: \
14567 ID = Intrinsic::x86_##NAME; \
14569 switch (BuiltinID) {
14570 default: llvm_unreachable(
"Unsupported intrinsic!");
14584 case X86::BI_xsetbv:
14585 ID = Intrinsic::x86_xsetbv;
14588 #undef INTRINSIC_X86_XSAVE_ID
14589 Value *Mhi = Builder.CreateTrunc(
14590 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
14591 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
14593 Ops.push_back(Mlo);
14594 return Builder.CreateCall(CGM.getIntrinsic(
ID), Ops);
14596 case X86::BI__builtin_ia32_xgetbv:
14597 case X86::BI_xgetbv:
14598 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_xgetbv), Ops);
14599 case X86::BI__builtin_ia32_storedqudi128_mask:
14600 case X86::BI__builtin_ia32_storedqusi128_mask:
14601 case X86::BI__builtin_ia32_storedquhi128_mask:
14602 case X86::BI__builtin_ia32_storedquqi128_mask:
14603 case X86::BI__builtin_ia32_storeupd128_mask:
14604 case X86::BI__builtin_ia32_storeups128_mask:
14605 case X86::BI__builtin_ia32_storedqudi256_mask:
14606 case X86::BI__builtin_ia32_storedqusi256_mask:
14607 case X86::BI__builtin_ia32_storedquhi256_mask:
14608 case X86::BI__builtin_ia32_storedquqi256_mask:
14609 case X86::BI__builtin_ia32_storeupd256_mask:
14610 case X86::BI__builtin_ia32_storeups256_mask:
14611 case X86::BI__builtin_ia32_storedqudi512_mask:
14612 case X86::BI__builtin_ia32_storedqusi512_mask:
14613 case X86::BI__builtin_ia32_storedquhi512_mask:
14614 case X86::BI__builtin_ia32_storedquqi512_mask:
14615 case X86::BI__builtin_ia32_storeupd512_mask:
14616 case X86::BI__builtin_ia32_storeups512_mask:
14619 case X86::BI__builtin_ia32_storesh128_mask:
14620 case X86::BI__builtin_ia32_storess128_mask:
14621 case X86::BI__builtin_ia32_storesd128_mask:
14624 case X86::BI__builtin_ia32_vpopcntb_128:
14625 case X86::BI__builtin_ia32_vpopcntd_128:
14626 case X86::BI__builtin_ia32_vpopcntq_128:
14627 case X86::BI__builtin_ia32_vpopcntw_128:
14628 case X86::BI__builtin_ia32_vpopcntb_256:
14629 case X86::BI__builtin_ia32_vpopcntd_256:
14630 case X86::BI__builtin_ia32_vpopcntq_256:
14631 case X86::BI__builtin_ia32_vpopcntw_256:
14632 case X86::BI__builtin_ia32_vpopcntb_512:
14633 case X86::BI__builtin_ia32_vpopcntd_512:
14634 case X86::BI__builtin_ia32_vpopcntq_512:
14635 case X86::BI__builtin_ia32_vpopcntw_512: {
14636 llvm::Type *ResultType = ConvertType(E->
getType());
14637 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
14638 return Builder.CreateCall(F, Ops);
14640 case X86::BI__builtin_ia32_cvtmask2b128:
14641 case X86::BI__builtin_ia32_cvtmask2b256:
14642 case X86::BI__builtin_ia32_cvtmask2b512:
14643 case X86::BI__builtin_ia32_cvtmask2w128:
14644 case X86::BI__builtin_ia32_cvtmask2w256:
14645 case X86::BI__builtin_ia32_cvtmask2w512:
14646 case X86::BI__builtin_ia32_cvtmask2d128:
14647 case X86::BI__builtin_ia32_cvtmask2d256:
14648 case X86::BI__builtin_ia32_cvtmask2d512:
14649 case X86::BI__builtin_ia32_cvtmask2q128:
14650 case X86::BI__builtin_ia32_cvtmask2q256:
14651 case X86::BI__builtin_ia32_cvtmask2q512:
14654 case X86::BI__builtin_ia32_cvtb2mask128:
14655 case X86::BI__builtin_ia32_cvtb2mask256:
14656 case X86::BI__builtin_ia32_cvtb2mask512:
14657 case X86::BI__builtin_ia32_cvtw2mask128:
14658 case X86::BI__builtin_ia32_cvtw2mask256:
14659 case X86::BI__builtin_ia32_cvtw2mask512:
14660 case X86::BI__builtin_ia32_cvtd2mask128:
14661 case X86::BI__builtin_ia32_cvtd2mask256:
14662 case X86::BI__builtin_ia32_cvtd2mask512:
14663 case X86::BI__builtin_ia32_cvtq2mask128:
14664 case X86::BI__builtin_ia32_cvtq2mask256:
14665 case X86::BI__builtin_ia32_cvtq2mask512:
14668 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
14669 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
14670 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
14671 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
14672 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
14673 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
14675 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
14676 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
14677 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
14678 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
14679 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
14680 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
14683 case X86::BI__builtin_ia32_vfmaddss3:
14684 case X86::BI__builtin_ia32_vfmaddsd3:
14685 case X86::BI__builtin_ia32_vfmaddsh3_mask:
14686 case X86::BI__builtin_ia32_vfmaddss3_mask:
14687 case X86::BI__builtin_ia32_vfmaddsd3_mask:
14689 case X86::BI__builtin_ia32_vfmaddss:
14690 case X86::BI__builtin_ia32_vfmaddsd:
14692 Constant::getNullValue(Ops[0]->getType()));
14693 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
14694 case X86::BI__builtin_ia32_vfmaddss3_maskz:
14695 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
14697 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
14698 case X86::BI__builtin_ia32_vfmaddss3_mask3:
14699 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
14701 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
14702 case X86::BI__builtin_ia32_vfmsubss3_mask3:
14703 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
14706 case X86::BI__builtin_ia32_vfmaddph:
14707 case X86::BI__builtin_ia32_vfmaddps:
14708 case X86::BI__builtin_ia32_vfmaddpd:
14709 case X86::BI__builtin_ia32_vfmaddph256:
14710 case X86::BI__builtin_ia32_vfmaddps256:
14711 case X86::BI__builtin_ia32_vfmaddpd256:
14712 case X86::BI__builtin_ia32_vfmaddph512_mask:
14713 case X86::BI__builtin_ia32_vfmaddph512_maskz:
14714 case X86::BI__builtin_ia32_vfmaddph512_mask3:
14715 case X86::BI__builtin_ia32_vfmaddps512_mask:
14716 case X86::BI__builtin_ia32_vfmaddps512_maskz:
14717 case X86::BI__builtin_ia32_vfmaddps512_mask3:
14718 case X86::BI__builtin_ia32_vfmsubps512_mask3:
14719 case X86::BI__builtin_ia32_vfmaddpd512_mask:
14720 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
14721 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
14722 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
14723 case X86::BI__builtin_ia32_vfmsubph512_mask3:
14725 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
14726 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14727 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14728 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14729 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
14730 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14731 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14732 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14733 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14734 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14735 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14736 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14739 case X86::BI__builtin_ia32_movdqa32store128_mask:
14740 case X86::BI__builtin_ia32_movdqa64store128_mask:
14741 case X86::BI__builtin_ia32_storeaps128_mask:
14742 case X86::BI__builtin_ia32_storeapd128_mask:
14743 case X86::BI__builtin_ia32_movdqa32store256_mask:
14744 case X86::BI__builtin_ia32_movdqa64store256_mask:
14745 case X86::BI__builtin_ia32_storeaps256_mask:
14746 case X86::BI__builtin_ia32_storeapd256_mask:
14747 case X86::BI__builtin_ia32_movdqa32store512_mask:
14748 case X86::BI__builtin_ia32_movdqa64store512_mask:
14749 case X86::BI__builtin_ia32_storeaps512_mask:
14750 case X86::BI__builtin_ia32_storeapd512_mask:
14753 getContext().getTypeAlignInChars(E->
getArg(1)->
getType()).getAsAlign());
14755 case X86::BI__builtin_ia32_loadups128_mask:
14756 case X86::BI__builtin_ia32_loadups256_mask:
14757 case X86::BI__builtin_ia32_loadups512_mask:
14758 case X86::BI__builtin_ia32_loadupd128_mask:
14759 case X86::BI__builtin_ia32_loadupd256_mask:
14760 case X86::BI__builtin_ia32_loadupd512_mask:
14761 case X86::BI__builtin_ia32_loaddquqi128_mask:
14762 case X86::BI__builtin_ia32_loaddquqi256_mask:
14763 case X86::BI__builtin_ia32_loaddquqi512_mask:
14764 case X86::BI__builtin_ia32_loaddquhi128_mask:
14765 case X86::BI__builtin_ia32_loaddquhi256_mask:
14766 case X86::BI__builtin_ia32_loaddquhi512_mask:
14767 case X86::BI__builtin_ia32_loaddqusi128_mask:
14768 case X86::BI__builtin_ia32_loaddqusi256_mask:
14769 case X86::BI__builtin_ia32_loaddqusi512_mask:
14770 case X86::BI__builtin_ia32_loaddqudi128_mask:
14771 case X86::BI__builtin_ia32_loaddqudi256_mask:
14772 case X86::BI__builtin_ia32_loaddqudi512_mask:
14775 case X86::BI__builtin_ia32_loadsh128_mask:
14776 case X86::BI__builtin_ia32_loadss128_mask:
14777 case X86::BI__builtin_ia32_loadsd128_mask:
14780 case X86::BI__builtin_ia32_loadaps128_mask:
14781 case X86::BI__builtin_ia32_loadaps256_mask:
14782 case X86::BI__builtin_ia32_loadaps512_mask:
14783 case X86::BI__builtin_ia32_loadapd128_mask:
14784 case X86::BI__builtin_ia32_loadapd256_mask:
14785 case X86::BI__builtin_ia32_loadapd512_mask:
14786 case X86::BI__builtin_ia32_movdqa32load128_mask:
14787 case X86::BI__builtin_ia32_movdqa32load256_mask:
14788 case X86::BI__builtin_ia32_movdqa32load512_mask:
14789 case X86::BI__builtin_ia32_movdqa64load128_mask:
14790 case X86::BI__builtin_ia32_movdqa64load256_mask:
14791 case X86::BI__builtin_ia32_movdqa64load512_mask:
14794 getContext().getTypeAlignInChars(E->
getArg(1)->
getType()).getAsAlign());
14796 case X86::BI__builtin_ia32_expandloaddf128_mask:
14797 case X86::BI__builtin_ia32_expandloaddf256_mask:
14798 case X86::BI__builtin_ia32_expandloaddf512_mask:
14799 case X86::BI__builtin_ia32_expandloadsf128_mask:
14800 case X86::BI__builtin_ia32_expandloadsf256_mask:
14801 case X86::BI__builtin_ia32_expandloadsf512_mask:
14802 case X86::BI__builtin_ia32_expandloaddi128_mask:
14803 case X86::BI__builtin_ia32_expandloaddi256_mask:
14804 case X86::BI__builtin_ia32_expandloaddi512_mask:
14805 case X86::BI__builtin_ia32_expandloadsi128_mask:
14806 case X86::BI__builtin_ia32_expandloadsi256_mask:
14807 case X86::BI__builtin_ia32_expandloadsi512_mask:
14808 case X86::BI__builtin_ia32_expandloadhi128_mask:
14809 case X86::BI__builtin_ia32_expandloadhi256_mask:
14810 case X86::BI__builtin_ia32_expandloadhi512_mask:
14811 case X86::BI__builtin_ia32_expandloadqi128_mask:
14812 case X86::BI__builtin_ia32_expandloadqi256_mask:
14813 case X86::BI__builtin_ia32_expandloadqi512_mask:
14816 case X86::BI__builtin_ia32_compressstoredf128_mask:
14817 case X86::BI__builtin_ia32_compressstoredf256_mask:
14818 case X86::BI__builtin_ia32_compressstoredf512_mask:
14819 case X86::BI__builtin_ia32_compressstoresf128_mask:
14820 case X86::BI__builtin_ia32_compressstoresf256_mask:
14821 case X86::BI__builtin_ia32_compressstoresf512_mask:
14822 case X86::BI__builtin_ia32_compressstoredi128_mask:
14823 case X86::BI__builtin_ia32_compressstoredi256_mask:
14824 case X86::BI__builtin_ia32_compressstoredi512_mask:
14825 case X86::BI__builtin_ia32_compressstoresi128_mask:
14826 case X86::BI__builtin_ia32_compressstoresi256_mask:
14827 case X86::BI__builtin_ia32_compressstoresi512_mask:
14828 case X86::BI__builtin_ia32_compressstorehi128_mask:
14829 case X86::BI__builtin_ia32_compressstorehi256_mask:
14830 case X86::BI__builtin_ia32_compressstorehi512_mask:
14831 case X86::BI__builtin_ia32_compressstoreqi128_mask:
14832 case X86::BI__builtin_ia32_compressstoreqi256_mask:
14833 case X86::BI__builtin_ia32_compressstoreqi512_mask:
14836 case X86::BI__builtin_ia32_expanddf128_mask:
14837 case X86::BI__builtin_ia32_expanddf256_mask:
14838 case X86::BI__builtin_ia32_expanddf512_mask:
14839 case X86::BI__builtin_ia32_expandsf128_mask:
14840 case X86::BI__builtin_ia32_expandsf256_mask:
14841 case X86::BI__builtin_ia32_expandsf512_mask:
14842 case X86::BI__builtin_ia32_expanddi128_mask:
14843 case X86::BI__builtin_ia32_expanddi256_mask:
14844 case X86::BI__builtin_ia32_expanddi512_mask:
14845 case X86::BI__builtin_ia32_expandsi128_mask:
14846 case X86::BI__builtin_ia32_expandsi256_mask:
14847 case X86::BI__builtin_ia32_expandsi512_mask:
14848 case X86::BI__builtin_ia32_expandhi128_mask:
14849 case X86::BI__builtin_ia32_expandhi256_mask:
14850 case X86::BI__builtin_ia32_expandhi512_mask:
14851 case X86::BI__builtin_ia32_expandqi128_mask:
14852 case X86::BI__builtin_ia32_expandqi256_mask:
14853 case X86::BI__builtin_ia32_expandqi512_mask:
14856 case X86::BI__builtin_ia32_compressdf128_mask:
14857 case X86::BI__builtin_ia32_compressdf256_mask:
14858 case X86::BI__builtin_ia32_compressdf512_mask:
14859 case X86::BI__builtin_ia32_compresssf128_mask:
14860 case X86::BI__builtin_ia32_compresssf256_mask:
14861 case X86::BI__builtin_ia32_compresssf512_mask:
14862 case X86::BI__builtin_ia32_compressdi128_mask:
14863 case X86::BI__builtin_ia32_compressdi256_mask:
14864 case X86::BI__builtin_ia32_compressdi512_mask:
14865 case X86::BI__builtin_ia32_compresssi128_mask:
14866 case X86::BI__builtin_ia32_compresssi256_mask:
14867 case X86::BI__builtin_ia32_compresssi512_mask:
14868 case X86::BI__builtin_ia32_compresshi128_mask:
14869 case X86::BI__builtin_ia32_compresshi256_mask:
14870 case X86::BI__builtin_ia32_compresshi512_mask:
14871 case X86::BI__builtin_ia32_compressqi128_mask:
14872 case X86::BI__builtin_ia32_compressqi256_mask:
14873 case X86::BI__builtin_ia32_compressqi512_mask:
14876 case X86::BI__builtin_ia32_gather3div2df:
14877 case X86::BI__builtin_ia32_gather3div2di:
14878 case X86::BI__builtin_ia32_gather3div4df:
14879 case X86::BI__builtin_ia32_gather3div4di:
14880 case X86::BI__builtin_ia32_gather3div4sf:
14881 case X86::BI__builtin_ia32_gather3div4si:
14882 case X86::BI__builtin_ia32_gather3div8sf:
14883 case X86::BI__builtin_ia32_gather3div8si:
14884 case X86::BI__builtin_ia32_gather3siv2df:
14885 case X86::BI__builtin_ia32_gather3siv2di:
14886 case X86::BI__builtin_ia32_gather3siv4df:
14887 case X86::BI__builtin_ia32_gather3siv4di:
14888 case X86::BI__builtin_ia32_gather3siv4sf:
14889 case X86::BI__builtin_ia32_gather3siv4si:
14890 case X86::BI__builtin_ia32_gather3siv8sf:
14891 case X86::BI__builtin_ia32_gather3siv8si:
14892 case X86::BI__builtin_ia32_gathersiv8df:
14893 case X86::BI__builtin_ia32_gathersiv16sf:
14894 case X86::BI__builtin_ia32_gatherdiv8df:
14895 case X86::BI__builtin_ia32_gatherdiv16sf:
14896 case X86::BI__builtin_ia32_gathersiv8di:
14897 case X86::BI__builtin_ia32_gathersiv16si:
14898 case X86::BI__builtin_ia32_gatherdiv8di:
14899 case X86::BI__builtin_ia32_gatherdiv16si: {
14901 switch (BuiltinID) {
14902 default: llvm_unreachable(
"Unexpected builtin");
14903 case X86::BI__builtin_ia32_gather3div2df:
14904 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
14906 case X86::BI__builtin_ia32_gather3div2di:
14907 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
14909 case X86::BI__builtin_ia32_gather3div4df:
14910 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
14912 case X86::BI__builtin_ia32_gather3div4di:
14913 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
14915 case X86::BI__builtin_ia32_gather3div4sf:
14916 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
14918 case X86::BI__builtin_ia32_gather3div4si:
14919 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
14921 case X86::BI__builtin_ia32_gather3div8sf:
14922 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
14924 case X86::BI__builtin_ia32_gather3div8si:
14925 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
14927 case X86::BI__builtin_ia32_gather3siv2df:
14928 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
14930 case X86::BI__builtin_ia32_gather3siv2di:
14931 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
14933 case X86::BI__builtin_ia32_gather3siv4df:
14934 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
14936 case X86::BI__builtin_ia32_gather3siv4di:
14937 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
14939 case X86::BI__builtin_ia32_gather3siv4sf:
14940 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
14942 case X86::BI__builtin_ia32_gather3siv4si:
14943 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
14945 case X86::BI__builtin_ia32_gather3siv8sf:
14946 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
14948 case X86::BI__builtin_ia32_gather3siv8si:
14949 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
14951 case X86::BI__builtin_ia32_gathersiv8df:
14952 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
14954 case X86::BI__builtin_ia32_gathersiv16sf:
14955 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
14957 case X86::BI__builtin_ia32_gatherdiv8df:
14958 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
14960 case X86::BI__builtin_ia32_gatherdiv16sf:
14961 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
14963 case X86::BI__builtin_ia32_gathersiv8di:
14964 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
14966 case X86::BI__builtin_ia32_gathersiv16si:
14967 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
14969 case X86::BI__builtin_ia32_gatherdiv8di:
14970 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
14972 case X86::BI__builtin_ia32_gatherdiv16si:
14973 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
14978 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
14979 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
14981 Function *Intr = CGM.getIntrinsic(IID);
14982 return Builder.CreateCall(Intr, Ops);
14985 case X86::BI__builtin_ia32_scattersiv8df:
14986 case X86::BI__builtin_ia32_scattersiv16sf:
14987 case X86::BI__builtin_ia32_scatterdiv8df:
14988 case X86::BI__builtin_ia32_scatterdiv16sf:
14989 case X86::BI__builtin_ia32_scattersiv8di:
14990 case X86::BI__builtin_ia32_scattersiv16si:
14991 case X86::BI__builtin_ia32_scatterdiv8di:
14992 case X86::BI__builtin_ia32_scatterdiv16si:
14993 case X86::BI__builtin_ia32_scatterdiv2df:
14994 case X86::BI__builtin_ia32_scatterdiv2di:
14995 case X86::BI__builtin_ia32_scatterdiv4df:
14996 case X86::BI__builtin_ia32_scatterdiv4di:
14997 case X86::BI__builtin_ia32_scatterdiv4sf:
14998 case X86::BI__builtin_ia32_scatterdiv4si:
14999 case X86::BI__builtin_ia32_scatterdiv8sf:
15000 case X86::BI__builtin_ia32_scatterdiv8si:
15001 case X86::BI__builtin_ia32_scattersiv2df:
15002 case X86::BI__builtin_ia32_scattersiv2di:
15003 case X86::BI__builtin_ia32_scattersiv4df:
15004 case X86::BI__builtin_ia32_scattersiv4di:
15005 case X86::BI__builtin_ia32_scattersiv4sf:
15006 case X86::BI__builtin_ia32_scattersiv4si:
15007 case X86::BI__builtin_ia32_scattersiv8sf:
15008 case X86::BI__builtin_ia32_scattersiv8si: {
15010 switch (BuiltinID) {
15011 default: llvm_unreachable(
"Unexpected builtin");
15012 case X86::BI__builtin_ia32_scattersiv8df:
15013 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
15015 case X86::BI__builtin_ia32_scattersiv16sf:
15016 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
15018 case X86::BI__builtin_ia32_scatterdiv8df:
15019 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
15021 case X86::BI__builtin_ia32_scatterdiv16sf:
15022 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
15024 case X86::BI__builtin_ia32_scattersiv8di:
15025 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
15027 case X86::BI__builtin_ia32_scattersiv16si:
15028 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
15030 case X86::BI__builtin_ia32_scatterdiv8di:
15031 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
15033 case X86::BI__builtin_ia32_scatterdiv16si:
15034 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
15036 case X86::BI__builtin_ia32_scatterdiv2df:
15037 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
15039 case X86::BI__builtin_ia32_scatterdiv2di:
15040 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
15042 case X86::BI__builtin_ia32_scatterdiv4df:
15043 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
15045 case X86::BI__builtin_ia32_scatterdiv4di:
15046 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
15048 case X86::BI__builtin_ia32_scatterdiv4sf:
15049 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
15051 case X86::BI__builtin_ia32_scatterdiv4si:
15052 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
15054 case X86::BI__builtin_ia32_scatterdiv8sf:
15055 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
15057 case X86::BI__builtin_ia32_scatterdiv8si:
15058 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
15060 case X86::BI__builtin_ia32_scattersiv2df:
15061 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
15063 case X86::BI__builtin_ia32_scattersiv2di:
15064 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
15066 case X86::BI__builtin_ia32_scattersiv4df:
15067 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
15069 case X86::BI__builtin_ia32_scattersiv4di:
15070 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
15072 case X86::BI__builtin_ia32_scattersiv4sf:
15073 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
15075 case X86::BI__builtin_ia32_scattersiv4si:
15076 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
15078 case X86::BI__builtin_ia32_scattersiv8sf:
15079 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
15081 case X86::BI__builtin_ia32_scattersiv8si:
15082 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
15087 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
15088 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
15090 Function *Intr = CGM.getIntrinsic(IID);
15091 return Builder.CreateCall(Intr, Ops);
15094 case X86::BI__builtin_ia32_vextractf128_pd256:
15095 case X86::BI__builtin_ia32_vextractf128_ps256:
15096 case X86::BI__builtin_ia32_vextractf128_si256:
15097 case X86::BI__builtin_ia32_extract128i256:
15098 case X86::BI__builtin_ia32_extractf64x4_mask:
15099 case X86::BI__builtin_ia32_extractf32x4_mask:
15100 case X86::BI__builtin_ia32_extracti64x4_mask:
15101 case X86::BI__builtin_ia32_extracti32x4_mask:
15102 case X86::BI__builtin_ia32_extractf32x8_mask:
15103 case X86::BI__builtin_ia32_extracti32x8_mask:
15104 case X86::BI__builtin_ia32_extractf32x4_256_mask:
15105 case X86::BI__builtin_ia32_extracti32x4_256_mask:
15106 case X86::BI__builtin_ia32_extractf64x2_256_mask:
15107 case X86::BI__builtin_ia32_extracti64x2_256_mask:
15108 case X86::BI__builtin_ia32_extractf64x2_512_mask:
15109 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
15110 auto *DstTy = cast<llvm::FixedVectorType>(ConvertType(E->
getType()));
15111 unsigned NumElts = DstTy->getNumElements();
15112 unsigned SrcNumElts =
15113 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15114 unsigned SubVectors = SrcNumElts / NumElts;
15115 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15116 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15117 Index &= SubVectors - 1;
15121 for (
unsigned i = 0; i != NumElts; ++i)
15122 Indices[i] = i + Index;
15124 Value *Res = Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15127 if (Ops.size() == 4)
15132 case X86::BI__builtin_ia32_vinsertf128_pd256:
15133 case X86::BI__builtin_ia32_vinsertf128_ps256:
15134 case X86::BI__builtin_ia32_vinsertf128_si256:
15135 case X86::BI__builtin_ia32_insert128i256:
15136 case X86::BI__builtin_ia32_insertf64x4:
15137 case X86::BI__builtin_ia32_insertf32x4:
15138 case X86::BI__builtin_ia32_inserti64x4:
15139 case X86::BI__builtin_ia32_inserti32x4:
15140 case X86::BI__builtin_ia32_insertf32x8:
15141 case X86::BI__builtin_ia32_inserti32x8:
15142 case X86::BI__builtin_ia32_insertf32x4_256:
15143 case X86::BI__builtin_ia32_inserti32x4_256:
15144 case X86::BI__builtin_ia32_insertf64x2_256:
15145 case X86::BI__builtin_ia32_inserti64x2_256:
15146 case X86::BI__builtin_ia32_insertf64x2_512:
15147 case X86::BI__builtin_ia32_inserti64x2_512: {
15148 unsigned DstNumElts =
15149 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15150 unsigned SrcNumElts =
15151 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
15152 unsigned SubVectors = DstNumElts / SrcNumElts;
15153 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15154 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15155 Index &= SubVectors - 1;
15156 Index *= SrcNumElts;
15159 for (
unsigned i = 0; i != DstNumElts; ++i)
15160 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
15162 Value *Op1 = Builder.CreateShuffleVector(
15163 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
15165 for (
unsigned i = 0; i != DstNumElts; ++i) {
15166 if (i >= Index && i < (Index + SrcNumElts))
15167 Indices[i] = (i - Index) + DstNumElts;
15172 return Builder.CreateShuffleVector(Ops[0], Op1,
15173 ArrayRef(Indices, DstNumElts),
"insert");
15175 case X86::BI__builtin_ia32_pmovqd512_mask:
15176 case X86::BI__builtin_ia32_pmovwb512_mask: {
15177 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15180 case X86::BI__builtin_ia32_pmovdb512_mask:
15181 case X86::BI__builtin_ia32_pmovdw512_mask:
15182 case X86::BI__builtin_ia32_pmovqw512_mask: {
15183 if (
const auto *C = dyn_cast<Constant>(Ops[2]))
15184 if (C->isAllOnesValue())
15185 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15188 switch (BuiltinID) {
15189 default: llvm_unreachable(
"Unsupported intrinsic!");
15190 case X86::BI__builtin_ia32_pmovdb512_mask:
15191 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
15193 case X86::BI__builtin_ia32_pmovdw512_mask:
15194 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
15196 case X86::BI__builtin_ia32_pmovqw512_mask:
15197 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
15201 Function *Intr = CGM.getIntrinsic(IID);
15202 return Builder.CreateCall(Intr, Ops);
15204 case X86::BI__builtin_ia32_pblendw128:
15205 case X86::BI__builtin_ia32_blendpd:
15206 case X86::BI__builtin_ia32_blendps:
15207 case X86::BI__builtin_ia32_blendpd256:
15208 case X86::BI__builtin_ia32_blendps256:
15209 case X86::BI__builtin_ia32_pblendw256:
15210 case X86::BI__builtin_ia32_pblendd128:
15211 case X86::BI__builtin_ia32_pblendd256: {
15213 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15214 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15219 for (
unsigned i = 0; i != NumElts; ++i)
15220 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
15222 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15223 ArrayRef(Indices, NumElts),
"blend");
15225 case X86::BI__builtin_ia32_pshuflw:
15226 case X86::BI__builtin_ia32_pshuflw256:
15227 case X86::BI__builtin_ia32_pshuflw512: {
15228 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15229 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15230 unsigned NumElts = Ty->getNumElements();
15233 Imm = (Imm & 0xff) * 0x01010101;
15236 for (
unsigned l = 0; l != NumElts; l += 8) {
15237 for (
unsigned i = 0; i != 4; ++i) {
15238 Indices[l + i] = l + (Imm & 3);
15241 for (
unsigned i = 4; i != 8; ++i)
15242 Indices[l + i] = l + i;
15245 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15248 case X86::BI__builtin_ia32_pshufhw:
15249 case X86::BI__builtin_ia32_pshufhw256:
15250 case X86::BI__builtin_ia32_pshufhw512: {
15251 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15252 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15253 unsigned NumElts = Ty->getNumElements();
15256 Imm = (Imm & 0xff) * 0x01010101;
15259 for (
unsigned l = 0; l != NumElts; l += 8) {
15260 for (
unsigned i = 0; i != 4; ++i)
15261 Indices[l + i] = l + i;
15262 for (
unsigned i = 4; i != 8; ++i) {
15263 Indices[l + i] = l + 4 + (Imm & 3);
15268 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15271 case X86::BI__builtin_ia32_pshufd:
15272 case X86::BI__builtin_ia32_pshufd256:
15273 case X86::BI__builtin_ia32_pshufd512:
15274 case X86::BI__builtin_ia32_vpermilpd:
15275 case X86::BI__builtin_ia32_vpermilps:
15276 case X86::BI__builtin_ia32_vpermilpd256:
15277 case X86::BI__builtin_ia32_vpermilps256:
15278 case X86::BI__builtin_ia32_vpermilpd512:
15279 case X86::BI__builtin_ia32_vpermilps512: {
15280 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15281 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15282 unsigned NumElts = Ty->getNumElements();
15283 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15284 unsigned NumLaneElts = NumElts / NumLanes;
15287 Imm = (Imm & 0xff) * 0x01010101;
15290 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15291 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15292 Indices[i + l] = (Imm % NumLaneElts) + l;
15293 Imm /= NumLaneElts;
15297 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15300 case X86::BI__builtin_ia32_shufpd:
15301 case X86::BI__builtin_ia32_shufpd256:
15302 case X86::BI__builtin_ia32_shufpd512:
15303 case X86::BI__builtin_ia32_shufps:
15304 case X86::BI__builtin_ia32_shufps256:
15305 case X86::BI__builtin_ia32_shufps512: {
15306 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15307 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15308 unsigned NumElts = Ty->getNumElements();
15309 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15310 unsigned NumLaneElts = NumElts / NumLanes;
15313 Imm = (Imm & 0xff) * 0x01010101;
15316 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15317 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15318 unsigned Index = Imm % NumLaneElts;
15319 Imm /= NumLaneElts;
15320 if (i >= (NumLaneElts / 2))
15322 Indices[l + i] = l + Index;
15326 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15327 ArrayRef(Indices, NumElts),
"shufp");
15329 case X86::BI__builtin_ia32_permdi256:
15330 case X86::BI__builtin_ia32_permdf256:
15331 case X86::BI__builtin_ia32_permdi512:
15332 case X86::BI__builtin_ia32_permdf512: {
15333 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15334 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15335 unsigned NumElts = Ty->getNumElements();
15339 for (
unsigned l = 0; l != NumElts; l += 4)
15340 for (
unsigned i = 0; i != 4; ++i)
15341 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
15343 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15346 case X86::BI__builtin_ia32_palignr128:
15347 case X86::BI__builtin_ia32_palignr256:
15348 case X86::BI__builtin_ia32_palignr512: {
15349 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15352 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15353 assert(NumElts % 16 == 0);
15357 if (ShiftVal >= 32)
15358 return llvm::Constant::getNullValue(ConvertType(E->
getType()));
15362 if (ShiftVal > 16) {
15365 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
15370 for (
unsigned l = 0; l != NumElts; l += 16) {
15371 for (
unsigned i = 0; i != 16; ++i) {
15372 unsigned Idx = ShiftVal + i;
15374 Idx += NumElts - 16;
15375 Indices[l + i] = Idx + l;
15379 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15380 ArrayRef(Indices, NumElts),
"palignr");
15382 case X86::BI__builtin_ia32_alignd128:
15383 case X86::BI__builtin_ia32_alignd256:
15384 case X86::BI__builtin_ia32_alignd512:
15385 case X86::BI__builtin_ia32_alignq128:
15386 case X86::BI__builtin_ia32_alignq256:
15387 case X86::BI__builtin_ia32_alignq512: {
15389 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15390 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15393 ShiftVal &= NumElts - 1;
15396 for (
unsigned i = 0; i != NumElts; ++i)
15397 Indices[i] = i + ShiftVal;
15399 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15400 ArrayRef(Indices, NumElts),
"valign");
15402 case X86::BI__builtin_ia32_shuf_f32x4_256:
15403 case X86::BI__builtin_ia32_shuf_f64x2_256:
15404 case X86::BI__builtin_ia32_shuf_i32x4_256:
15405 case X86::BI__builtin_ia32_shuf_i64x2_256:
15406 case X86::BI__builtin_ia32_shuf_f32x4:
15407 case X86::BI__builtin_ia32_shuf_f64x2:
15408 case X86::BI__builtin_ia32_shuf_i32x4:
15409 case X86::BI__builtin_ia32_shuf_i64x2: {
15410 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15411 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15412 unsigned NumElts = Ty->getNumElements();
15413 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
15414 unsigned NumLaneElts = NumElts / NumLanes;
15417 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15418 unsigned Index = (Imm % NumLanes) * NumLaneElts;
15420 if (l >= (NumElts / 2))
15422 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15423 Indices[l + i] = Index + i;
15427 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15428 ArrayRef(Indices, NumElts),
"shuf");
15431 case X86::BI__builtin_ia32_vperm2f128_pd256:
15432 case X86::BI__builtin_ia32_vperm2f128_ps256:
15433 case X86::BI__builtin_ia32_vperm2f128_si256:
15434 case X86::BI__builtin_ia32_permti256: {
15435 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15437 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15446 for (
unsigned l = 0; l != 2; ++l) {
15448 if (Imm & (1 << ((l * 4) + 3)))
15449 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
15450 else if (Imm & (1 << ((l * 4) + 1)))
15451 OutOps[l] = Ops[1];
15453 OutOps[l] = Ops[0];
15455 for (
unsigned i = 0; i != NumElts/2; ++i) {
15457 unsigned Idx = (l * NumElts) + i;
15460 if (Imm & (1 << (l * 4)))
15462 Indices[(l * (NumElts/2)) + i] = Idx;
15466 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
15467 ArrayRef(Indices, NumElts),
"vperm");
15470 case X86::BI__builtin_ia32_pslldqi128_byteshift:
15471 case X86::BI__builtin_ia32_pslldqi256_byteshift:
15472 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
15473 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15474 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15476 unsigned NumElts = ResultType->getNumElements() * 8;
15479 if (ShiftVal >= 16)
15480 return llvm::Constant::getNullValue(ResultType);
15484 for (
unsigned l = 0; l != NumElts; l += 16) {
15485 for (
unsigned i = 0; i != 16; ++i) {
15486 unsigned Idx = NumElts + i - ShiftVal;
15487 if (Idx < NumElts) Idx -= NumElts - 16;
15488 Indices[l + i] = Idx + l;
15492 auto *VecTy = llvm::FixedVectorType::get(Int8Ty, NumElts);
15493 Value *
Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
15494 Value *Zero = llvm::Constant::getNullValue(VecTy);
15495 Value *SV = Builder.CreateShuffleVector(
15497 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
15499 case X86::BI__builtin_ia32_psrldqi128_byteshift:
15500 case X86::BI__builtin_ia32_psrldqi256_byteshift:
15501 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
15502 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15503 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15505 unsigned NumElts = ResultType->getNumElements() * 8;
15508 if (ShiftVal >= 16)
15509 return llvm::Constant::getNullValue(ResultType);
15513 for (
unsigned l = 0; l != NumElts; l += 16) {
15514 for (
unsigned i = 0; i != 16; ++i) {
15515 unsigned Idx = i + ShiftVal;
15516 if (Idx >= 16) Idx += NumElts - 16;
15517 Indices[l + i] = Idx + l;
15521 auto *VecTy = llvm::FixedVectorType::get(Int8Ty, NumElts);
15522 Value *
Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
15523 Value *Zero = llvm::Constant::getNullValue(VecTy);
15524 Value *SV = Builder.CreateShuffleVector(
15526 return Builder.CreateBitCast(SV, ResultType,
"cast");
15528 case X86::BI__builtin_ia32_kshiftliqi:
15529 case X86::BI__builtin_ia32_kshiftlihi:
15530 case X86::BI__builtin_ia32_kshiftlisi:
15531 case X86::BI__builtin_ia32_kshiftlidi: {
15532 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15533 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15535 if (ShiftVal >= NumElts)
15536 return llvm::Constant::getNullValue(Ops[0]->getType());
15541 for (
unsigned i = 0; i != NumElts; ++i)
15542 Indices[i] = NumElts + i - ShiftVal;
15544 Value *Zero = llvm::Constant::getNullValue(In->getType());
15545 Value *SV = Builder.CreateShuffleVector(
15546 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
15547 return Builder.CreateBitCast(SV, Ops[0]->getType());
15549 case X86::BI__builtin_ia32_kshiftriqi:
15550 case X86::BI__builtin_ia32_kshiftrihi:
15551 case X86::BI__builtin_ia32_kshiftrisi:
15552 case X86::BI__builtin_ia32_kshiftridi: {
15553 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15554 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15556 if (ShiftVal >= NumElts)
15557 return llvm::Constant::getNullValue(Ops[0]->getType());
15562 for (
unsigned i = 0; i != NumElts; ++i)
15563 Indices[i] = i + ShiftVal;
15565 Value *Zero = llvm::Constant::getNullValue(In->getType());
15566 Value *SV = Builder.CreateShuffleVector(
15567 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
15568 return Builder.CreateBitCast(SV, Ops[0]->getType());
15570 case X86::BI__builtin_ia32_movnti:
15571 case X86::BI__builtin_ia32_movnti64:
15572 case X86::BI__builtin_ia32_movntsd:
15573 case X86::BI__builtin_ia32_movntss: {
15574 llvm::MDNode *
Node = llvm::MDNode::get(
15575 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
15577 Value *Ptr = Ops[0];
15578 Value *Src = Ops[1];
15581 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
15582 BuiltinID == X86::BI__builtin_ia32_movntss)
15583 Src = Builder.CreateExtractElement(Src, (
uint64_t)0,
"extract");
15586 StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, Ptr);
15587 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
15588 SI->setAlignment(llvm::Align(1));
15592 case X86::BI__builtin_ia32_vprotb:
15593 case X86::BI__builtin_ia32_vprotw:
15594 case X86::BI__builtin_ia32_vprotd:
15595 case X86::BI__builtin_ia32_vprotq:
15596 case X86::BI__builtin_ia32_vprotbi:
15597 case X86::BI__builtin_ia32_vprotwi:
15598 case X86::BI__builtin_ia32_vprotdi:
15599 case X86::BI__builtin_ia32_vprotqi:
15600 case X86::BI__builtin_ia32_prold128:
15601 case X86::BI__builtin_ia32_prold256:
15602 case X86::BI__builtin_ia32_prold512:
15603 case X86::BI__builtin_ia32_prolq128:
15604 case X86::BI__builtin_ia32_prolq256:
15605 case X86::BI__builtin_ia32_prolq512:
15606 case X86::BI__builtin_ia32_prolvd128:
15607 case X86::BI__builtin_ia32_prolvd256:
15608 case X86::BI__builtin_ia32_prolvd512:
15609 case X86::BI__builtin_ia32_prolvq128:
15610 case X86::BI__builtin_ia32_prolvq256:
15611 case X86::BI__builtin_ia32_prolvq512:
15613 case X86::BI__builtin_ia32_prord128:
15614 case X86::BI__builtin_ia32_prord256:
15615 case X86::BI__builtin_ia32_prord512:
15616 case X86::BI__builtin_ia32_prorq128:
15617 case X86::BI__builtin_ia32_prorq256:
15618 case X86::BI__builtin_ia32_prorq512:
15619 case X86::BI__builtin_ia32_prorvd128:
15620 case X86::BI__builtin_ia32_prorvd256:
15621 case X86::BI__builtin_ia32_prorvd512:
15622 case X86::BI__builtin_ia32_prorvq128:
15623 case X86::BI__builtin_ia32_prorvq256:
15624 case X86::BI__builtin_ia32_prorvq512:
15626 case X86::BI__builtin_ia32_selectb_128:
15627 case X86::BI__builtin_ia32_selectb_256:
15628 case X86::BI__builtin_ia32_selectb_512:
15629 case X86::BI__builtin_ia32_selectw_128:
15630 case X86::BI__builtin_ia32_selectw_256:
15631 case X86::BI__builtin_ia32_selectw_512:
15632 case X86::BI__builtin_ia32_selectd_128:
15633 case X86::BI__builtin_ia32_selectd_256:
15634 case X86::BI__builtin_ia32_selectd_512:
15635 case X86::BI__builtin_ia32_selectq_128:
15636 case X86::BI__builtin_ia32_selectq_256:
15637 case X86::BI__builtin_ia32_selectq_512:
15638 case X86::BI__builtin_ia32_selectph_128:
15639 case X86::BI__builtin_ia32_selectph_256:
15640 case X86::BI__builtin_ia32_selectph_512:
15641 case X86::BI__builtin_ia32_selectpbf_128:
15642 case X86::BI__builtin_ia32_selectpbf_256:
15643 case X86::BI__builtin_ia32_selectpbf_512:
15644 case X86::BI__builtin_ia32_selectps_128:
15645 case X86::BI__builtin_ia32_selectps_256:
15646 case X86::BI__builtin_ia32_selectps_512:
15647 case X86::BI__builtin_ia32_selectpd_128:
15648 case X86::BI__builtin_ia32_selectpd_256:
15649 case X86::BI__builtin_ia32_selectpd_512:
15651 case X86::BI__builtin_ia32_selectsh_128:
15652 case X86::BI__builtin_ia32_selectsbf_128:
15653 case X86::BI__builtin_ia32_selectss_128:
15654 case X86::BI__builtin_ia32_selectsd_128: {
15655 Value *A = Builder.CreateExtractElement(Ops[1], (
uint64_t)0);
15656 Value *B = Builder.CreateExtractElement(Ops[2], (
uint64_t)0);
15658 return Builder.CreateInsertElement(Ops[1], A, (
uint64_t)0);
15660 case X86::BI__builtin_ia32_cmpb128_mask:
15661 case X86::BI__builtin_ia32_cmpb256_mask:
15662 case X86::BI__builtin_ia32_cmpb512_mask:
15663 case X86::BI__builtin_ia32_cmpw128_mask:
15664 case X86::BI__builtin_ia32_cmpw256_mask:
15665 case X86::BI__builtin_ia32_cmpw512_mask:
15666 case X86::BI__builtin_ia32_cmpd128_mask:
15667 case X86::BI__builtin_ia32_cmpd256_mask:
15668 case X86::BI__builtin_ia32_cmpd512_mask:
15669 case X86::BI__builtin_ia32_cmpq128_mask:
15670 case X86::BI__builtin_ia32_cmpq256_mask:
15671 case X86::BI__builtin_ia32_cmpq512_mask: {
15672 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15675 case X86::BI__builtin_ia32_ucmpb128_mask:
15676 case X86::BI__builtin_ia32_ucmpb256_mask:
15677 case X86::BI__builtin_ia32_ucmpb512_mask:
15678 case X86::BI__builtin_ia32_ucmpw128_mask:
15679 case X86::BI__builtin_ia32_ucmpw256_mask:
15680 case X86::BI__builtin_ia32_ucmpw512_mask:
15681 case X86::BI__builtin_ia32_ucmpd128_mask:
15682 case X86::BI__builtin_ia32_ucmpd256_mask:
15683 case X86::BI__builtin_ia32_ucmpd512_mask:
15684 case X86::BI__builtin_ia32_ucmpq128_mask:
15685 case X86::BI__builtin_ia32_ucmpq256_mask:
15686 case X86::BI__builtin_ia32_ucmpq512_mask: {
15687 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15690 case X86::BI__builtin_ia32_vpcomb:
15691 case X86::BI__builtin_ia32_vpcomw:
15692 case X86::BI__builtin_ia32_vpcomd:
15693 case X86::BI__builtin_ia32_vpcomq:
15695 case X86::BI__builtin_ia32_vpcomub:
15696 case X86::BI__builtin_ia32_vpcomuw:
15697 case X86::BI__builtin_ia32_vpcomud:
15698 case X86::BI__builtin_ia32_vpcomuq:
15701 case X86::BI__builtin_ia32_kortestcqi:
15702 case X86::BI__builtin_ia32_kortestchi:
15703 case X86::BI__builtin_ia32_kortestcsi:
15704 case X86::BI__builtin_ia32_kortestcdi: {
15706 Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
15707 Value *Cmp = Builder.CreateICmpEQ(
Or, C);
15708 return Builder.CreateZExt(Cmp, ConvertType(E->
getType()));
15710 case X86::BI__builtin_ia32_kortestzqi:
15711 case X86::BI__builtin_ia32_kortestzhi:
15712 case X86::BI__builtin_ia32_kortestzsi:
15713 case X86::BI__builtin_ia32_kortestzdi: {
15715 Value *C = llvm::Constant::getNullValue(Ops[0]->getType());
15716 Value *Cmp = Builder.CreateICmpEQ(
Or, C);
15717 return Builder.CreateZExt(Cmp, ConvertType(E->
getType()));
15720 case X86::BI__builtin_ia32_ktestcqi:
15721 case X86::BI__builtin_ia32_ktestzqi:
15722 case X86::BI__builtin_ia32_ktestchi:
15723 case X86::BI__builtin_ia32_ktestzhi:
15724 case X86::BI__builtin_ia32_ktestcsi:
15725 case X86::BI__builtin_ia32_ktestzsi:
15726 case X86::BI__builtin_ia32_ktestcdi:
15727 case X86::BI__builtin_ia32_ktestzdi: {
15729 switch (BuiltinID) {
15730 default: llvm_unreachable(
"Unsupported intrinsic!");
15731 case X86::BI__builtin_ia32_ktestcqi:
15732 IID = Intrinsic::x86_avx512_ktestc_b;
15734 case X86::BI__builtin_ia32_ktestzqi:
15735 IID = Intrinsic::x86_avx512_ktestz_b;
15737 case X86::BI__builtin_ia32_ktestchi:
15738 IID = Intrinsic::x86_avx512_ktestc_w;
15740 case X86::BI__builtin_ia32_ktestzhi:
15741 IID = Intrinsic::x86_avx512_ktestz_w;
15743 case X86::BI__builtin_ia32_ktestcsi:
15744 IID = Intrinsic::x86_avx512_ktestc_d;
15746 case X86::BI__builtin_ia32_ktestzsi:
15747 IID = Intrinsic::x86_avx512_ktestz_d;
15749 case X86::BI__builtin_ia32_ktestcdi:
15750 IID = Intrinsic::x86_avx512_ktestc_q;
15752 case X86::BI__builtin_ia32_ktestzdi:
15753 IID = Intrinsic::x86_avx512_ktestz_q;
15757 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15760 Function *Intr = CGM.getIntrinsic(IID);
15761 return Builder.CreateCall(Intr, {LHS, RHS});
15764 case X86::BI__builtin_ia32_kaddqi:
15765 case X86::BI__builtin_ia32_kaddhi:
15766 case X86::BI__builtin_ia32_kaddsi:
15767 case X86::BI__builtin_ia32_kadddi: {
15769 switch (BuiltinID) {
15770 default: llvm_unreachable(
"Unsupported intrinsic!");
15771 case X86::BI__builtin_ia32_kaddqi:
15772 IID = Intrinsic::x86_avx512_kadd_b;
15774 case X86::BI__builtin_ia32_kaddhi:
15775 IID = Intrinsic::x86_avx512_kadd_w;
15777 case X86::BI__builtin_ia32_kaddsi:
15778 IID = Intrinsic::x86_avx512_kadd_d;
15780 case X86::BI__builtin_ia32_kadddi:
15781 IID = Intrinsic::x86_avx512_kadd_q;
15785 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15788 Function *Intr = CGM.getIntrinsic(IID);
15789 Value *Res = Builder.CreateCall(Intr, {LHS, RHS});
15790 return Builder.CreateBitCast(Res, Ops[0]->getType());
15792 case X86::BI__builtin_ia32_kandqi:
15793 case X86::BI__builtin_ia32_kandhi:
15794 case X86::BI__builtin_ia32_kandsi:
15795 case X86::BI__builtin_ia32_kanddi:
15797 case X86::BI__builtin_ia32_kandnqi:
15798 case X86::BI__builtin_ia32_kandnhi:
15799 case X86::BI__builtin_ia32_kandnsi:
15800 case X86::BI__builtin_ia32_kandndi:
15802 case X86::BI__builtin_ia32_korqi:
15803 case X86::BI__builtin_ia32_korhi:
15804 case X86::BI__builtin_ia32_korsi:
15805 case X86::BI__builtin_ia32_kordi:
15807 case X86::BI__builtin_ia32_kxnorqi:
15808 case X86::BI__builtin_ia32_kxnorhi:
15809 case X86::BI__builtin_ia32_kxnorsi:
15810 case X86::BI__builtin_ia32_kxnordi:
15812 case X86::BI__builtin_ia32_kxorqi:
15813 case X86::BI__builtin_ia32_kxorhi:
15814 case X86::BI__builtin_ia32_kxorsi:
15815 case X86::BI__builtin_ia32_kxordi:
15817 case X86::BI__builtin_ia32_knotqi:
15818 case X86::BI__builtin_ia32_knothi:
15819 case X86::BI__builtin_ia32_knotsi:
15820 case X86::BI__builtin_ia32_knotdi: {
15821 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15823 return Builder.CreateBitCast(Builder.CreateNot(Res),
15824 Ops[0]->getType());
15826 case X86::BI__builtin_ia32_kmovb:
15827 case X86::BI__builtin_ia32_kmovw:
15828 case X86::BI__builtin_ia32_kmovd:
15829 case X86::BI__builtin_ia32_kmovq: {
15833 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15835 return Builder.CreateBitCast(Res, Ops[0]->getType());
15838 case X86::BI__builtin_ia32_kunpckdi:
15839 case X86::BI__builtin_ia32_kunpcksi:
15840 case X86::BI__builtin_ia32_kunpckhi: {
15841 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15845 for (
unsigned i = 0; i != NumElts; ++i)
15850 LHS = Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
15851 RHS = Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
15855 Builder.CreateShuffleVector(RHS, LHS,
ArrayRef(Indices, NumElts));
15856 return Builder.CreateBitCast(Res, Ops[0]->getType());
15859 case X86::BI__builtin_ia32_vplzcntd_128:
15860 case X86::BI__builtin_ia32_vplzcntd_256:
15861 case X86::BI__builtin_ia32_vplzcntd_512:
15862 case X86::BI__builtin_ia32_vplzcntq_128:
15863 case X86::BI__builtin_ia32_vplzcntq_256:
15864 case X86::BI__builtin_ia32_vplzcntq_512: {
15865 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
15866 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(
false)});
15868 case X86::BI__builtin_ia32_sqrtss:
15869 case X86::BI__builtin_ia32_sqrtsd: {
15870 Value *A = Builder.CreateExtractElement(Ops[0], (
uint64_t)0);
15872 if (Builder.getIsFPConstrained()) {
15874 F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
15876 A = Builder.CreateConstrainedFPCall(F, {A});
15879 A = Builder.CreateCall(F, {A});
15881 return Builder.CreateInsertElement(Ops[0], A, (
uint64_t)0);
15883 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15884 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15885 case X86::BI__builtin_ia32_sqrtss_round_mask: {
15886 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
15892 switch (BuiltinID) {
15894 llvm_unreachable(
"Unsupported intrinsic!");
15895 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15896 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
15898 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15899 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
15901 case X86::BI__builtin_ia32_sqrtss_round_mask:
15902 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
15905 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
15907 Value *A = Builder.CreateExtractElement(Ops[1], (
uint64_t)0);
15909 if (Builder.getIsFPConstrained()) {
15911 F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
15913 A = Builder.CreateConstrainedFPCall(F, A);
15916 A = Builder.CreateCall(F, A);
15918 Value *Src = Builder.CreateExtractElement(Ops[2], (
uint64_t)0);
15920 return Builder.CreateInsertElement(Ops[0], A, (
uint64_t)0);
15922 case X86::BI__builtin_ia32_sqrtpd256:
15923 case X86::BI__builtin_ia32_sqrtpd:
15924 case X86::BI__builtin_ia32_sqrtps256:
15925 case X86::BI__builtin_ia32_sqrtps:
15926 case X86::BI__builtin_ia32_sqrtph256:
15927 case X86::BI__builtin_ia32_sqrtph:
15928 case X86::BI__builtin_ia32_sqrtph512:
15929 case X86::BI__builtin_ia32_sqrtps512:
15930 case X86::BI__builtin_ia32_sqrtpd512: {
15931 if (Ops.size() == 2) {
15932 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15938 switch (BuiltinID) {
15940 llvm_unreachable(
"Unsupported intrinsic!");
15941 case X86::BI__builtin_ia32_sqrtph512:
15942 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
15944 case X86::BI__builtin_ia32_sqrtps512:
15945 IID = Intrinsic::x86_avx512_sqrt_ps_512;
15947 case X86::BI__builtin_ia32_sqrtpd512:
15948 IID = Intrinsic::x86_avx512_sqrt_pd_512;
15951 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
15954 if (Builder.getIsFPConstrained()) {
15956 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
15957 Ops[0]->getType());
15958 return Builder.CreateConstrainedFPCall(F, Ops[0]);
15961 return Builder.CreateCall(F, Ops[0]);
15965 case X86::BI__builtin_ia32_pmuludq128:
15966 case X86::BI__builtin_ia32_pmuludq256:
15967 case X86::BI__builtin_ia32_pmuludq512:
15970 case X86::BI__builtin_ia32_pmuldq128:
15971 case X86::BI__builtin_ia32_pmuldq256:
15972 case X86::BI__builtin_ia32_pmuldq512:
15975 case X86::BI__builtin_ia32_pternlogd512_mask:
15976 case X86::BI__builtin_ia32_pternlogq512_mask:
15977 case X86::BI__builtin_ia32_pternlogd128_mask:
15978 case X86::BI__builtin_ia32_pternlogd256_mask:
15979 case X86::BI__builtin_ia32_pternlogq128_mask:
15980 case X86::BI__builtin_ia32_pternlogq256_mask:
15983 case X86::BI__builtin_ia32_pternlogd512_maskz:
15984 case X86::BI__builtin_ia32_pternlogq512_maskz:
15985 case X86::BI__builtin_ia32_pternlogd128_maskz:
15986 case X86::BI__builtin_ia32_pternlogd256_maskz:
15987 case X86::BI__builtin_ia32_pternlogq128_maskz:
15988 case X86::BI__builtin_ia32_pternlogq256_maskz:
15991 case X86::BI__builtin_ia32_vpshldd128:
15992 case X86::BI__builtin_ia32_vpshldd256:
15993 case X86::BI__builtin_ia32_vpshldd512:
15994 case X86::BI__builtin_ia32_vpshldq128:
15995 case X86::BI__builtin_ia32_vpshldq256:
15996 case X86::BI__builtin_ia32_vpshldq512:
15997 case X86::BI__builtin_ia32_vpshldw128:
15998 case X86::BI__builtin_ia32_vpshldw256:
15999 case X86::BI__builtin_ia32_vpshldw512:
16002 case X86::BI__builtin_ia32_vpshrdd128:
16003 case X86::BI__builtin_ia32_vpshrdd256:
16004 case X86::BI__builtin_ia32_vpshrdd512:
16005 case X86::BI__builtin_ia32_vpshrdq128:
16006 case X86::BI__builtin_ia32_vpshrdq256:
16007 case X86::BI__builtin_ia32_vpshrdq512:
16008 case X86::BI__builtin_ia32_vpshrdw128:
16009 case X86::BI__builtin_ia32_vpshrdw256:
16010 case X86::BI__builtin_ia32_vpshrdw512:
16014 case X86::BI__builtin_ia32_vpshldvd128:
16015 case X86::BI__builtin_ia32_vpshldvd256:
16016 case X86::BI__builtin_ia32_vpshldvd512:
16017 case X86::BI__builtin_ia32_vpshldvq128:
16018 case X86::BI__builtin_ia32_vpshldvq256:
16019 case X86::BI__builtin_ia32_vpshldvq512:
16020 case X86::BI__builtin_ia32_vpshldvw128:
16021 case X86::BI__builtin_ia32_vpshldvw256:
16022 case X86::BI__builtin_ia32_vpshldvw512:
16025 case X86::BI__builtin_ia32_vpshrdvd128:
16026 case X86::BI__builtin_ia32_vpshrdvd256:
16027 case X86::BI__builtin_ia32_vpshrdvd512:
16028 case X86::BI__builtin_ia32_vpshrdvq128:
16029 case X86::BI__builtin_ia32_vpshrdvq256:
16030 case X86::BI__builtin_ia32_vpshrdvq512:
16031 case X86::BI__builtin_ia32_vpshrdvw128:
16032 case X86::BI__builtin_ia32_vpshrdvw256:
16033 case X86::BI__builtin_ia32_vpshrdvw512:
16038 case X86::BI__builtin_ia32_reduce_fadd_pd512:
16039 case X86::BI__builtin_ia32_reduce_fadd_ps512:
16040 case X86::BI__builtin_ia32_reduce_fadd_ph512:
16041 case X86::BI__builtin_ia32_reduce_fadd_ph256:
16042 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
16044 CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType());
16045 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
16046 Builder.getFastMathFlags().setAllowReassoc();
16047 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16049 case X86::BI__builtin_ia32_reduce_fmul_pd512:
16050 case X86::BI__builtin_ia32_reduce_fmul_ps512:
16051 case X86::BI__builtin_ia32_reduce_fmul_ph512:
16052 case X86::BI__builtin_ia32_reduce_fmul_ph256:
16053 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
16055 CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType());
16056 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
16057 Builder.getFastMathFlags().setAllowReassoc();
16058 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16060 case X86::BI__builtin_ia32_reduce_fmax_pd512:
16061 case X86::BI__builtin_ia32_reduce_fmax_ps512:
16062 case X86::BI__builtin_ia32_reduce_fmax_ph512:
16063 case X86::BI__builtin_ia32_reduce_fmax_ph256:
16064 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
16066 CGM.getIntrinsic(Intrinsic::vector_reduce_fmax, Ops[0]->getType());
16067 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
16068 Builder.getFastMathFlags().setNoNaNs();
16069 return Builder.CreateCall(F, {Ops[0]});
16071 case X86::BI__builtin_ia32_reduce_fmin_pd512:
16072 case X86::BI__builtin_ia32_reduce_fmin_ps512:
16073 case X86::BI__builtin_ia32_reduce_fmin_ph512:
16074 case X86::BI__builtin_ia32_reduce_fmin_ph256:
16075 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
16077 CGM.getIntrinsic(Intrinsic::vector_reduce_fmin, Ops[0]->getType());
16078 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
16079 Builder.getFastMathFlags().setNoNaNs();
16080 return Builder.CreateCall(F, {Ops[0]});
16084 case X86::BI__builtin_ia32_pswapdsf:
16085 case X86::BI__builtin_ia32_pswapdsi: {
16086 llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
16087 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy,
"cast");
16088 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd);
16089 return Builder.CreateCall(F, Ops,
"pswapd");
16091 case X86::BI__builtin_ia32_rdrand16_step:
16092 case X86::BI__builtin_ia32_rdrand32_step:
16093 case X86::BI__builtin_ia32_rdrand64_step:
16094 case X86::BI__builtin_ia32_rdseed16_step:
16095 case X86::BI__builtin_ia32_rdseed32_step:
16096 case X86::BI__builtin_ia32_rdseed64_step: {
16098 switch (BuiltinID) {
16099 default: llvm_unreachable(
"Unsupported intrinsic!");
16100 case X86::BI__builtin_ia32_rdrand16_step:
16101 ID = Intrinsic::x86_rdrand_16;
16103 case X86::BI__builtin_ia32_rdrand32_step:
16104 ID = Intrinsic::x86_rdrand_32;
16106 case X86::BI__builtin_ia32_rdrand64_step:
16107 ID = Intrinsic::x86_rdrand_64;
16109 case X86::BI__builtin_ia32_rdseed16_step:
16110 ID = Intrinsic::x86_rdseed_16;
16112 case X86::BI__builtin_ia32_rdseed32_step:
16113 ID = Intrinsic::x86_rdseed_32;
16115 case X86::BI__builtin_ia32_rdseed64_step:
16116 ID = Intrinsic::x86_rdseed_64;
16120 Value *Call = Builder.CreateCall(CGM.getIntrinsic(
ID));
16121 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
16123 return Builder.CreateExtractValue(Call, 1);
16125 case X86::BI__builtin_ia32_addcarryx_u32:
16126 case X86::BI__builtin_ia32_addcarryx_u64:
16127 case X86::BI__builtin_ia32_subborrow_u32:
16128 case X86::BI__builtin_ia32_subborrow_u64: {
16130 switch (BuiltinID) {
16131 default: llvm_unreachable(
"Unsupported intrinsic!");
16132 case X86::BI__builtin_ia32_addcarryx_u32:
16133 IID = Intrinsic::x86_addcarry_32;
16135 case X86::BI__builtin_ia32_addcarryx_u64:
16136 IID = Intrinsic::x86_addcarry_64;
16138 case X86::BI__builtin_ia32_subborrow_u32:
16139 IID = Intrinsic::x86_subborrow_32;
16141 case X86::BI__builtin_ia32_subborrow_u64:
16142 IID = Intrinsic::x86_subborrow_64;
16146 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID),
16147 { Ops[0], Ops[1], Ops[2] });
16148 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
16150 return Builder.CreateExtractValue(Call, 0);
16153 case X86::BI__builtin_ia32_fpclassps128_mask:
16154 case X86::BI__builtin_ia32_fpclassps256_mask:
16155 case X86::BI__builtin_ia32_fpclassps512_mask:
16156 case X86::BI__builtin_ia32_fpclassph128_mask:
16157 case X86::BI__builtin_ia32_fpclassph256_mask:
16158 case X86::BI__builtin_ia32_fpclassph512_mask:
16159 case X86::BI__builtin_ia32_fpclasspd128_mask:
16160 case X86::BI__builtin_ia32_fpclasspd256_mask:
16161 case X86::BI__builtin_ia32_fpclasspd512_mask: {
16163 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16164 Value *MaskIn = Ops[2];
16165 Ops.erase(&Ops[2]);
16168 switch (BuiltinID) {
16169 default: llvm_unreachable(
"Unsupported intrinsic!");
16170 case X86::BI__builtin_ia32_fpclassph128_mask:
16171 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
16173 case X86::BI__builtin_ia32_fpclassph256_mask:
16174 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
16176 case X86::BI__builtin_ia32_fpclassph512_mask:
16177 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
16179 case X86::BI__builtin_ia32_fpclassps128_mask:
16180 ID = Intrinsic::x86_avx512_fpclass_ps_128;
16182 case X86::BI__builtin_ia32_fpclassps256_mask:
16183 ID = Intrinsic::x86_avx512_fpclass_ps_256;
16185 case X86::BI__builtin_ia32_fpclassps512_mask:
16186 ID = Intrinsic::x86_avx512_fpclass_ps_512;
16188 case X86::BI__builtin_ia32_fpclasspd128_mask:
16189 ID = Intrinsic::x86_avx512_fpclass_pd_128;
16191 case X86::BI__builtin_ia32_fpclasspd256_mask:
16192 ID = Intrinsic::x86_avx512_fpclass_pd_256;
16194 case X86::BI__builtin_ia32_fpclasspd512_mask:
16195 ID = Intrinsic::x86_avx512_fpclass_pd_512;
16199 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(
ID), Ops);
16203 case X86::BI__builtin_ia32_vp2intersect_q_512:
16204 case X86::BI__builtin_ia32_vp2intersect_q_256:
16205 case X86::BI__builtin_ia32_vp2intersect_q_128:
16206 case X86::BI__builtin_ia32_vp2intersect_d_512:
16207 case X86::BI__builtin_ia32_vp2intersect_d_256:
16208 case X86::BI__builtin_ia32_vp2intersect_d_128: {
16210 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16213 switch (BuiltinID) {
16214 default: llvm_unreachable(
"Unsupported intrinsic!");
16215 case X86::BI__builtin_ia32_vp2intersect_q_512:
16216 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
16218 case X86::BI__builtin_ia32_vp2intersect_q_256:
16219 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
16221 case X86::BI__builtin_ia32_vp2intersect_q_128:
16222 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
16224 case X86::BI__builtin_ia32_vp2intersect_d_512:
16225 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
16227 case X86::BI__builtin_ia32_vp2intersect_d_256:
16228 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
16230 case X86::BI__builtin_ia32_vp2intersect_d_128:
16231 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
16235 Value *Call = Builder.CreateCall(CGM.getIntrinsic(
ID), {Ops[0], Ops[1]});
16236 Value *Result = Builder.CreateExtractValue(Call, 0);
16238 Builder.CreateDefaultAlignedStore(Result, Ops[2]);
16240 Result = Builder.CreateExtractValue(Call, 1);
16242 return Builder.CreateDefaultAlignedStore(Result, Ops[3]);
16245 case X86::BI__builtin_ia32_vpmultishiftqb128:
16246 case X86::BI__builtin_ia32_vpmultishiftqb256:
16247 case X86::BI__builtin_ia32_vpmultishiftqb512: {
16249 switch (BuiltinID) {
16250 default: llvm_unreachable(
"Unsupported intrinsic!");
16251 case X86::BI__builtin_ia32_vpmultishiftqb128:
16252 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
16254 case X86::BI__builtin_ia32_vpmultishiftqb256:
16255 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
16257 case X86::BI__builtin_ia32_vpmultishiftqb512:
16258 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
16262 return Builder.CreateCall(CGM.getIntrinsic(
ID), Ops);
16265 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16266 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16267 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
16269 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16270 Value *MaskIn = Ops[2];
16271 Ops.erase(&Ops[2]);
16274 switch (BuiltinID) {
16275 default: llvm_unreachable(
"Unsupported intrinsic!");
16276 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16277 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
16279 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16280 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
16282 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
16283 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
16287 Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(
ID), Ops);
16292 case X86::BI__builtin_ia32_cmpeqps:
16293 case X86::BI__builtin_ia32_cmpeqpd:
16294 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
16295 case X86::BI__builtin_ia32_cmpltps:
16296 case X86::BI__builtin_ia32_cmpltpd:
16297 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
16298 case X86::BI__builtin_ia32_cmpleps:
16299 case X86::BI__builtin_ia32_cmplepd:
16300 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
16301 case X86::BI__builtin_ia32_cmpunordps:
16302 case X86::BI__builtin_ia32_cmpunordpd:
16303 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
16304 case X86::BI__builtin_ia32_cmpneqps:
16305 case X86::BI__builtin_ia32_cmpneqpd:
16306 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
16307 case X86::BI__builtin_ia32_cmpnltps:
16308 case X86::BI__builtin_ia32_cmpnltpd:
16309 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
16310 case X86::BI__builtin_ia32_cmpnleps:
16311 case X86::BI__builtin_ia32_cmpnlepd:
16312 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
16313 case X86::BI__builtin_ia32_cmpordps:
16314 case X86::BI__builtin_ia32_cmpordpd:
16315 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
16316 case X86::BI__builtin_ia32_cmpph128_mask:
16317 case X86::BI__builtin_ia32_cmpph256_mask:
16318 case X86::BI__builtin_ia32_cmpph512_mask:
16319 case X86::BI__builtin_ia32_cmpps128_mask:
16320 case X86::BI__builtin_ia32_cmpps256_mask:
16321 case X86::BI__builtin_ia32_cmpps512_mask:
16322 case X86::BI__builtin_ia32_cmppd128_mask:
16323 case X86::BI__builtin_ia32_cmppd256_mask:
16324 case X86::BI__builtin_ia32_cmppd512_mask:
16327 case X86::BI__builtin_ia32_cmpps:
16328 case X86::BI__builtin_ia32_cmpps256:
16329 case X86::BI__builtin_ia32_cmppd:
16330 case X86::BI__builtin_ia32_cmppd256: {
16338 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
16343 FCmpInst::Predicate Pred;
16347 switch (CC & 0xf) {
16348 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
16349 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
16350 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
16351 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
16352 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
16353 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
16354 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
16355 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
16356 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
16357 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
16358 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
16359 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
16360 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
16361 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
16362 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
16363 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
16364 default: llvm_unreachable(
"Unhandled CC");
16369 IsSignaling = !IsSignaling;
16376 if (Builder.getIsFPConstrained() &&
16377 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
16381 switch (BuiltinID) {
16382 default: llvm_unreachable(
"Unexpected builtin");
16383 case X86::BI__builtin_ia32_cmpps:
16384 IID = Intrinsic::x86_sse_cmp_ps;
16386 case X86::BI__builtin_ia32_cmpps256:
16387 IID = Intrinsic::x86_avx_cmp_ps_256;
16389 case X86::BI__builtin_ia32_cmppd:
16390 IID = Intrinsic::x86_sse2_cmp_pd;
16392 case X86::BI__builtin_ia32_cmppd256:
16393 IID = Intrinsic::x86_avx_cmp_pd_256;
16395 case X86::BI__builtin_ia32_cmpph128_mask:
16396 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
16398 case X86::BI__builtin_ia32_cmpph256_mask:
16399 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
16401 case X86::BI__builtin_ia32_cmpph512_mask:
16402 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
16404 case X86::BI__builtin_ia32_cmpps512_mask:
16405 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
16407 case X86::BI__builtin_ia32_cmppd512_mask:
16408 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
16410 case X86::BI__builtin_ia32_cmpps128_mask:
16411 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
16413 case X86::BI__builtin_ia32_cmpps256_mask:
16414 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
16416 case X86::BI__builtin_ia32_cmppd128_mask:
16417 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
16419 case X86::BI__builtin_ia32_cmppd256_mask:
16420 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
16424 Function *Intr = CGM.getIntrinsic(IID);
16427 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16429 Value *Cmp = Builder.CreateCall(Intr, Ops);
16433 return Builder.CreateCall(Intr, Ops);
16444 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16447 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
16449 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
16453 return getVectorFCmpIR(Pred, IsSignaling);
16457 case X86::BI__builtin_ia32_cmpeqss:
16458 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
16459 case X86::BI__builtin_ia32_cmpltss:
16460 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
16461 case X86::BI__builtin_ia32_cmpless:
16462 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
16463 case X86::BI__builtin_ia32_cmpunordss:
16464 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
16465 case X86::BI__builtin_ia32_cmpneqss:
16466 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
16467 case X86::BI__builtin_ia32_cmpnltss:
16468 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
16469 case X86::BI__builtin_ia32_cmpnless:
16470 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
16471 case X86::BI__builtin_ia32_cmpordss:
16472 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
16473 case X86::BI__builtin_ia32_cmpeqsd:
16474 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
16475 case X86::BI__builtin_ia32_cmpltsd:
16476 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
16477 case X86::BI__builtin_ia32_cmplesd:
16478 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
16479 case X86::BI__builtin_ia32_cmpunordsd:
16480 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
16481 case X86::BI__builtin_ia32_cmpneqsd:
16482 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
16483 case X86::BI__builtin_ia32_cmpnltsd:
16484 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
16485 case X86::BI__builtin_ia32_cmpnlesd:
16486 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
16487 case X86::BI__builtin_ia32_cmpordsd:
16488 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
16491 case X86::BI__builtin_ia32_vcvtph2ps:
16492 case X86::BI__builtin_ia32_vcvtph2ps256:
16493 case X86::BI__builtin_ia32_vcvtph2ps_mask:
16494 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
16495 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
16501 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
16504 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
16505 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
16506 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
16508 case X86::BI__builtin_ia32_cvtsbf162ss_32:
16509 return Builder.CreateFPExt(Ops[0], Builder.getFloatTy());
16511 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16512 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
16514 switch (BuiltinID) {
16515 default: llvm_unreachable(
"Unsupported intrinsic!");
16516 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16517 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
16519 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
16520 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
16523 Value *Res = Builder.CreateCall(CGM.getIntrinsic(IID), Ops[0]);
16527 case X86::BI__cpuid:
16528 case X86::BI__cpuidex: {
16530 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
16531 ? EmitScalarExpr(E->
getArg(2))
16532 : llvm::ConstantInt::get(Int32Ty, 0);
16534 llvm::StructType *CpuidRetTy =
16535 llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty, Int32Ty);
16536 llvm::FunctionType *FTy =
16537 llvm::FunctionType::get(CpuidRetTy, {Int32Ty, Int32Ty},
false);
16539 StringRef
Asm, Constraints;
16540 if (getTarget().getTriple().getArch() == llvm::Triple::x86) {
16542 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
16545 Asm =
"xchgq %rbx, ${1:q}\n"
16547 "xchgq %rbx, ${1:q}";
16548 Constraints =
"={ax},=r,={cx},={dx},0,2";
16551 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
16553 Value *IACall = Builder.CreateCall(IA, {FuncId, SubFuncId});
16556 for (
unsigned i = 0; i < 4; i++) {
16557 Value *Extracted = Builder.CreateExtractValue(IACall, i);
16558 Value *StorePtr = Builder.CreateConstInBoundsGEP1_32(Int32Ty, BasePtr, i);
16559 Store = Builder.CreateAlignedStore(Extracted, StorePtr, getIntAlign());
16567 case X86::BI__emul:
16568 case X86::BI__emulu: {
16569 llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64);
16570 bool isSigned = (BuiltinID == X86::BI__emul);
16571 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned);
16572 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned);
16573 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
16575 case X86::BI__mulh:
16576 case X86::BI__umulh:
16577 case X86::BI_mul128:
16578 case X86::BI_umul128: {
16579 llvm::Type *ResType = ConvertType(E->
getType());
16580 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
16582 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
16583 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
16584 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
16586 Value *MulResult, *HigherBits;
16588 MulResult = Builder.CreateNSWMul(LHS, RHS);
16589 HigherBits = Builder.CreateAShr(MulResult, 64);
16591 MulResult = Builder.CreateNUWMul(LHS, RHS);
16592 HigherBits = Builder.CreateLShr(MulResult, 64);
16594 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
16596 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
16600 Builder.CreateStore(HigherBits, HighBitsAddress);
16601 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
16604 case X86::BI__faststorefence: {
16605 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16608 case X86::BI__shiftleft128:
16609 case X86::BI__shiftright128: {
16610 llvm::Function *F = CGM.getIntrinsic(
16611 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
16616 std::swap(Ops[0], Ops[1]);
16617 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
16618 return Builder.CreateCall(F, Ops);
16620 case X86::BI_ReadWriteBarrier:
16621 case X86::BI_ReadBarrier:
16622 case X86::BI_WriteBarrier: {
16623 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16624 llvm::SyncScope::SingleThread);
16627 case X86::BI_AddressOfReturnAddress: {
16629 CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
16630 return Builder.CreateCall(F);
16632 case X86::BI__stosb: {
16635 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1),
true);
16639 return EmitTrapCall(Intrinsic::trap);
16640 case X86::BI__int2c: {
16642 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
false);
16643 llvm::InlineAsm *IA =
16644 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
16645 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
16646 getLLVMContext(), llvm::AttributeList::FunctionIndex,
16647 llvm::Attribute::NoReturn);
16648 llvm::CallInst *CI = Builder.CreateCall(IA);
16649 CI->setAttributes(NoReturnAttr);
16652 case X86::BI__readfsbyte:
16653 case X86::BI__readfsword:
16654 case X86::BI__readfsdword:
16655 case X86::BI__readfsqword: {
16656 llvm::Type *IntTy = ConvertType(E->
getType());
16657 Value *Ptr = Builder.CreateIntToPtr(
16658 Ops[0], llvm::PointerType::get(getLLVMContext(), 257));
16659 LoadInst *
Load = Builder.CreateAlignedLoad(
16660 IntTy, Ptr, getContext().getTypeAlignInChars(E->
getType()));
16661 Load->setVolatile(
true);
16664 case X86::BI__readgsbyte:
16665 case X86::BI__readgsword:
16666 case X86::BI__readgsdword:
16667 case X86::BI__readgsqword: {
16668 llvm::Type *IntTy = ConvertType(E->
getType());
16669 Value *Ptr = Builder.CreateIntToPtr(
16670 Ops[0], llvm::PointerType::get(getLLVMContext(), 256));
16671 LoadInst *
Load = Builder.CreateAlignedLoad(
16672 IntTy, Ptr, getContext().getTypeAlignInChars(E->
getType()));
16673 Load->setVolatile(
true);
16676 case X86::BI__builtin_ia32_encodekey128_u32: {
16679 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1]});
16681 for (
int i = 0; i < 3; ++i) {
16682 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
16683 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[2], i * 16);
16684 Builder.CreateAlignedStore(Extract, Ptr, Align(1));
16687 return Builder.CreateExtractValue(Call, 0);
16689 case X86::BI__builtin_ia32_encodekey256_u32: {
16693 Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1], Ops[2]});
16695 for (
int i = 0; i < 4; ++i) {
16696 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
16697 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[3], i * 16);
16698 Builder.CreateAlignedStore(Extract, Ptr, Align(1));
16701 return Builder.CreateExtractValue(Call, 0);
16703 case X86::BI__builtin_ia32_aesenc128kl_u8:
16704 case X86::BI__builtin_ia32_aesdec128kl_u8:
16705 case X86::BI__builtin_ia32_aesenc256kl_u8:
16706 case X86::BI__builtin_ia32_aesdec256kl_u8: {
16708 StringRef BlockName;
16709 switch (BuiltinID) {
16711 llvm_unreachable(
"Unexpected builtin");
16712 case X86::BI__builtin_ia32_aesenc128kl_u8:
16713 IID = Intrinsic::x86_aesenc128kl;
16714 BlockName =
"aesenc128kl";
16716 case X86::BI__builtin_ia32_aesdec128kl_u8:
16717 IID = Intrinsic::x86_aesdec128kl;
16718 BlockName =
"aesdec128kl";
16720 case X86::BI__builtin_ia32_aesenc256kl_u8:
16721 IID = Intrinsic::x86_aesenc256kl;
16722 BlockName =
"aesenc256kl";
16724 case X86::BI__builtin_ia32_aesdec256kl_u8:
16725 IID = Intrinsic::x86_aesdec256kl;
16726 BlockName =
"aesdec256kl";
16730 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[1], Ops[2]});
16732 BasicBlock *NoError =
16733 createBasicBlock(BlockName +
"_no_error", this->CurFn);
16734 BasicBlock *Error = createBasicBlock(BlockName +
"_error", this->CurFn);
16735 BasicBlock *
End = createBasicBlock(BlockName +
"_end", this->CurFn);
16737 Value *
Ret = Builder.CreateExtractValue(Call, 0);
16738 Value *Succ = Builder.CreateTrunc(
Ret, Builder.getInt1Ty());
16739 Value *Out = Builder.CreateExtractValue(Call, 1);
16740 Builder.CreateCondBr(Succ, NoError, Error);
16742 Builder.SetInsertPoint(NoError);
16743 Builder.CreateDefaultAlignedStore(Out, Ops[0]);
16744 Builder.CreateBr(
End);
16746 Builder.SetInsertPoint(Error);
16747 Constant *Zero = llvm::Constant::getNullValue(Out->getType());
16748 Builder.CreateDefaultAlignedStore(Zero, Ops[0]);
16749 Builder.CreateBr(
End);
16751 Builder.SetInsertPoint(
End);
16752 return Builder.CreateExtractValue(Call, 0);
16754 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16755 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16756 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16757 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
16759 StringRef BlockName;
16760 switch (BuiltinID) {
16761 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16762 IID = Intrinsic::x86_aesencwide128kl;
16763 BlockName =
"aesencwide128kl";
16765 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16766 IID = Intrinsic::x86_aesdecwide128kl;
16767 BlockName =
"aesdecwide128kl";
16769 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16770 IID = Intrinsic::x86_aesencwide256kl;
16771 BlockName =
"aesencwide256kl";
16773 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
16774 IID = Intrinsic::x86_aesdecwide256kl;
16775 BlockName =
"aesdecwide256kl";
16779 llvm::Type *Ty = FixedVectorType::get(Builder.getInt64Ty(), 2);
16782 for (
int i = 0; i != 8; ++i) {
16783 Value *Ptr = Builder.CreateConstGEP1_32(Ty, Ops[1], i);
16784 InOps[i + 1] = Builder.CreateAlignedLoad(Ty, Ptr, Align(16));
16787 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), InOps);
16789 BasicBlock *NoError =
16790 createBasicBlock(BlockName +
"_no_error", this->CurFn);
16791 BasicBlock *Error = createBasicBlock(BlockName +
"_error", this->CurFn);
16792 BasicBlock *
End = createBasicBlock(BlockName +
"_end", this->CurFn);
16794 Value *
Ret = Builder.CreateExtractValue(Call, 0);
16795 Value *Succ = Builder.CreateTrunc(
Ret, Builder.getInt1Ty());
16796 Builder.CreateCondBr(Succ, NoError, Error);
16798 Builder.SetInsertPoint(NoError);
16799 for (
int i = 0; i != 8; ++i) {
16800 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
16801 Value *Ptr = Builder.CreateConstGEP1_32(Extract->
getType(), Ops[0], i);
16802 Builder.CreateAlignedStore(Extract, Ptr, Align(16));
16804 Builder.CreateBr(
End);
16806 Builder.SetInsertPoint(Error);
16807 for (
int i = 0; i != 8; ++i) {
16808 Value *Out = Builder.CreateExtractValue(Call, i + 1);
16809 Constant *Zero = llvm::Constant::getNullValue(Out->getType());
16810 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
16811 Builder.CreateAlignedStore(Zero, Ptr, Align(16));
16813 Builder.CreateBr(
End);
16815 Builder.SetInsertPoint(
End);
16816 return Builder.CreateExtractValue(Call, 0);
16818 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
16821 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
16823 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
16824 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
16825 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
16828 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
16831 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
16832 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16833 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16834 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
16835 Value *And = Builder.CreateAnd(Ops[3], llvm::ConstantInt::get(Int8Ty, 1));
16838 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
16841 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
16842 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16843 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16844 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
16845 static constexpr
int Mask[] = {0, 5, 6, 7};
16846 return Builder.CreateShuffleVector(Call, Ops[2], Mask);
16848 case X86::BI__builtin_ia32_prefetchi:
16849 return Builder.CreateCall(
16851 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
16852 llvm::ConstantInt::get(Int32Ty, 0)});
16872 #include "llvm/TargetParser/PPCTargetParser.def"
16873 auto GenAIXPPCBuiltinCpuExpr = [&](
unsigned SupportMethod,
unsigned FieldIdx,
16875 unsigned OpValue) ->
Value * {
16876 if (SupportMethod == AIX_BUILTIN_PPC_FALSE)
16877 return llvm::ConstantInt::getFalse(ConvertType(E->
getType()));
16879 if (SupportMethod == AIX_BUILTIN_PPC_TRUE)
16880 return llvm::ConstantInt::getTrue(ConvertType(E->
getType()));
16882 assert(SupportMethod <= USE_SYS_CONF &&
"Invalid value for SupportMethod.");
16883 assert((CompOp == COMP_EQ) &&
"Only equal comparisons are supported.");
16885 llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE);
16886 llvm::Constant *SysConf =
16887 CGM.CreateRuntimeVariable(STy,
"_system_configuration");
16890 llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0),
16891 ConstantInt::get(Int32Ty, FieldIdx)};
16893 llvm::Value *FieldValue = Builder.CreateGEP(STy, SysConf, Idxs);
16894 FieldValue = Builder.CreateAlignedLoad(Int32Ty, FieldValue,
16896 assert(FieldValue->getType()->isIntegerTy(32) &&
16897 "Only 32-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
16898 return Builder.CreateICmp(ICmpInst::ICMP_EQ, FieldValue,
16899 ConstantInt::get(Int32Ty, OpValue));
16902 switch (BuiltinID) {
16903 default:
return nullptr;
16905 case Builtin::BI__builtin_cpu_is: {
16907 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16908 llvm::Triple Triple = getTarget().getTriple();
16910 if (Triple.isOSAIX()) {
16911 unsigned IsCpuSupport, FieldIdx, CompareOp, CpuIdValue;
16912 typedef std::tuple<unsigned, unsigned, unsigned, unsigned> CPUType;
16913 std::tie(IsCpuSupport, FieldIdx, CompareOp, CpuIdValue) =
16914 static_cast<CPUType
>(StringSwitch<CPUType>(CPUStr)
16915 #define PPC_AIX_CPU(NAME, SUPPORT_MAGIC, INDEX, COMPARE_OP, VALUE) \
16916 .Case(NAME, {SUPPORT_MAGIC, INDEX, COMPARE_OP, VALUE})
16917 #include "llvm/TargetParser/PPCTargetParser.def"
16919 return GenAIXPPCBuiltinCpuExpr(IsCpuSupport, FieldIdx, CompareOp,
16923 assert(Triple.isOSLinux() &&
16924 "__builtin_cpu_is() is only supported for AIX and Linux.");
16925 unsigned NumCPUID = StringSwitch<unsigned>(CPUStr)
16926 #define PPC_LNX_CPU(Name, NumericID) .Case(Name, NumericID)
16927 #include "llvm/TargetParser/PPCTargetParser.def"
16929 assert(NumCPUID < -1U &&
"Invalid CPU name. Missed by SemaChecking?");
16930 Value *Op0 = llvm::ConstantInt::get(Int32Ty, PPC_FAWORD_CPUID);
16931 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_fixed_addr_ld);
16932 Value *TheCall = Builder.CreateCall(F, {Op0},
"cpu_is");
16933 return Builder.CreateICmpEQ(TheCall,
16934 llvm::ConstantInt::get(Int32Ty, NumCPUID));
16936 case Builtin::BI__builtin_cpu_supports: {
16937 unsigned FeatureWord;
16940 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16941 std::tie(FeatureWord, BitMask) =
16942 StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
16943 #define
PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \
16944 .Case(Name, {FA_WORD, Bitmask})
16945 #include
"llvm/TargetParser/PPCTargetParser.def"
16948 return Builder.getFalse();
16949 Value *Op0 = llvm::ConstantInt::get(Int32Ty, FeatureWord);
16950 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_fixed_addr_ld);
16951 Value *TheCall = Builder.CreateCall(F, {Op0},
"cpu_supports");
16953 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(Int32Ty, BitMask));
16954 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(Int32Ty));
16955 #undef PPC_FAWORD_HWCAP
16956 #undef PPC_FAWORD_HWCAP2
16957 #undef PPC_FAWORD_CPUID
16962 case PPC::BI__builtin_ppc_get_timebase:
16963 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
16966 case PPC::BI__builtin_altivec_lvx:
16967 case PPC::BI__builtin_altivec_lvxl:
16968 case PPC::BI__builtin_altivec_lvebx:
16969 case PPC::BI__builtin_altivec_lvehx:
16970 case PPC::BI__builtin_altivec_lvewx:
16971 case PPC::BI__builtin_altivec_lvsl:
16972 case PPC::BI__builtin_altivec_lvsr:
16973 case PPC::BI__builtin_vsx_lxvd2x:
16974 case PPC::BI__builtin_vsx_lxvw4x:
16975 case PPC::BI__builtin_vsx_lxvd2x_be:
16976 case PPC::BI__builtin_vsx_lxvw4x_be:
16977 case PPC::BI__builtin_vsx_lxvl:
16978 case PPC::BI__builtin_vsx_lxvll:
16981 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
16982 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
16983 if (!(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
16984 BuiltinID == PPC::BI__builtin_vsx_lxvll)) {
16985 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]);
16989 switch (BuiltinID) {
16990 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
16991 case PPC::BI__builtin_altivec_lvx:
16992 ID = Intrinsic::ppc_altivec_lvx;
16994 case PPC::BI__builtin_altivec_lvxl:
16995 ID = Intrinsic::ppc_altivec_lvxl;
16997 case PPC::BI__builtin_altivec_lvebx:
16998 ID = Intrinsic::ppc_altivec_lvebx;
17000 case PPC::BI__builtin_altivec_lvehx:
17001 ID = Intrinsic::ppc_altivec_lvehx;
17003 case PPC::BI__builtin_altivec_lvewx:
17004 ID = Intrinsic::ppc_altivec_lvewx;
17006 case PPC::BI__builtin_altivec_lvsl:
17007 ID = Intrinsic::ppc_altivec_lvsl;
17009 case PPC::BI__builtin_altivec_lvsr:
17010 ID = Intrinsic::ppc_altivec_lvsr;
17012 case PPC::BI__builtin_vsx_lxvd2x:
17013 ID = Intrinsic::ppc_vsx_lxvd2x;
17015 case PPC::BI__builtin_vsx_lxvw4x:
17016 ID = Intrinsic::ppc_vsx_lxvw4x;
17018 case PPC::BI__builtin_vsx_lxvd2x_be:
17019 ID = Intrinsic::ppc_vsx_lxvd2x_be;
17021 case PPC::BI__builtin_vsx_lxvw4x_be:
17022 ID = Intrinsic::ppc_vsx_lxvw4x_be;
17024 case PPC::BI__builtin_vsx_lxvl:
17025 ID = Intrinsic::ppc_vsx_lxvl;
17027 case PPC::BI__builtin_vsx_lxvll:
17028 ID = Intrinsic::ppc_vsx_lxvll;
17031 llvm::Function *F = CGM.getIntrinsic(
ID);
17032 return Builder.CreateCall(F, Ops,
"");
17036 case PPC::BI__builtin_altivec_stvx:
17037 case PPC::BI__builtin_altivec_stvxl:
17038 case PPC::BI__builtin_altivec_stvebx:
17039 case PPC::BI__builtin_altivec_stvehx:
17040 case PPC::BI__builtin_altivec_stvewx:
17041 case PPC::BI__builtin_vsx_stxvd2x:
17042 case PPC::BI__builtin_vsx_stxvw4x:
17043 case PPC::BI__builtin_vsx_stxvd2x_be:
17044 case PPC::BI__builtin_vsx_stxvw4x_be:
17045 case PPC::BI__builtin_vsx_stxvl:
17046 case PPC::BI__builtin_vsx_stxvll:
17049 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
17050 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
17051 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
17052 if (!(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
17053 BuiltinID == PPC::BI__builtin_vsx_stxvll)) {
17054 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]);
17058 switch (BuiltinID) {
17059 default: llvm_unreachable(
"Unsupported st intrinsic!");
17060 case PPC::BI__builtin_altivec_stvx:
17061 ID = Intrinsic::ppc_altivec_stvx;
17063 case PPC::BI__builtin_altivec_stvxl:
17064 ID = Intrinsic::ppc_altivec_stvxl;
17066 case PPC::BI__builtin_altivec_stvebx:
17067 ID = Intrinsic::ppc_altivec_stvebx;
17069 case PPC::BI__builtin_altivec_stvehx:
17070 ID = Intrinsic::ppc_altivec_stvehx;
17072 case PPC::BI__builtin_altivec_stvewx:
17073 ID = Intrinsic::ppc_altivec_stvewx;
17075 case PPC::BI__builtin_vsx_stxvd2x:
17076 ID = Intrinsic::ppc_vsx_stxvd2x;
17078 case PPC::BI__builtin_vsx_stxvw4x:
17079 ID = Intrinsic::ppc_vsx_stxvw4x;
17081 case PPC::BI__builtin_vsx_stxvd2x_be:
17082 ID = Intrinsic::ppc_vsx_stxvd2x_be;
17084 case PPC::BI__builtin_vsx_stxvw4x_be:
17085 ID = Intrinsic::ppc_vsx_stxvw4x_be;
17087 case PPC::BI__builtin_vsx_stxvl:
17088 ID = Intrinsic::ppc_vsx_stxvl;
17090 case PPC::BI__builtin_vsx_stxvll:
17091 ID = Intrinsic::ppc_vsx_stxvll;
17094 llvm::Function *F = CGM.getIntrinsic(
ID);
17095 return Builder.CreateCall(F, Ops,
"");
17097 case PPC::BI__builtin_vsx_ldrmb: {
17103 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17104 llvm::Type *ResTy = ConvertType(E->
getType());
17105 bool IsLE = getTarget().isLittleEndian();
17108 if (NumBytes == 16) {
17109 Value *BC = Builder.CreateBitCast(Op0, ResTy->getPointerTo());
17117 for (
int Idx = 0; Idx < 16; Idx++)
17118 RevMask.push_back(15 - Idx);
17119 return Builder.CreateShuffleVector(LD, LD, RevMask);
17122 llvm::Function *Lvx = CGM.getIntrinsic(Intrinsic::ppc_altivec_lvx);
17123 llvm::Function *Lvs = CGM.getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
17124 : Intrinsic::ppc_altivec_lvsl);
17125 llvm::Function *Vperm = CGM.getIntrinsic(Intrinsic::ppc_altivec_vperm);
17126 Value *HiMem = Builder.CreateGEP(
17127 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
17128 Value *LoLd = Builder.CreateCall(Lvx, Op0,
"ld.lo");
17129 Value *HiLd = Builder.CreateCall(Lvx, HiMem,
"ld.hi");
17130 Value *Mask1 = Builder.CreateCall(Lvs, Op0,
"mask1");
17132 Op0 = IsLE ? HiLd : LoLd;
17133 Op1 = IsLE ? LoLd : HiLd;
17134 Value *AllElts = Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
17135 Constant *Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
17139 for (
int Idx = 0; Idx < 16; Idx++) {
17140 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
17141 : 16 - (NumBytes - Idx);
17142 Consts.push_back(Val);
17144 return Builder.CreateShuffleVector(Builder.CreateBitCast(AllElts, ResTy),
17148 for (
int Idx = 0; Idx < 16; Idx++)
17149 Consts.push_back(Builder.getInt8(NumBytes + Idx));
17150 Value *Mask2 = ConstantVector::get(Consts);
17151 return Builder.CreateBitCast(
17152 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
17154 case PPC::BI__builtin_vsx_strmb: {
17158 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17159 bool IsLE = getTarget().isLittleEndian();
17160 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
17164 Value *StVec = Op2;
17167 for (
int Idx = 0; Idx < 16; Idx++)
17168 RevMask.push_back(15 - Idx);
17169 StVec = Builder.CreateShuffleVector(Op2, Op2, RevMask);
17171 return Builder.CreateStore(
17174 auto *ConvTy = Int64Ty;
17175 unsigned NumElts = 0;
17178 llvm_unreachable(
"width for stores must be a power of 2");
17196 Value *Vec = Builder.CreateBitCast(
17197 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
17199 Builder.CreateGEP(Int8Ty, Op0, ConstantInt::get(Int64Ty,
Offset));
17200 Value *Elt = Builder.CreateExtractElement(Vec, EltNo);
17201 if (IsLE && Width > 1) {
17202 Function *F = CGM.getIntrinsic(Intrinsic::bswap, ConvTy);
17203 Elt = Builder.CreateCall(F, Elt);
17205 return Builder.CreateStore(
17208 unsigned Stored = 0;
17209 unsigned RemainingBytes = NumBytes;
17211 if (NumBytes == 16)
17212 return StoreSubVec(16, 0, 0);
17213 if (NumBytes >= 8) {
17214 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
17215 RemainingBytes -= 8;
17218 if (RemainingBytes >= 4) {
17219 Result = StoreSubVec(4, NumBytes - Stored - 4,
17220 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
17221 RemainingBytes -= 4;
17224 if (RemainingBytes >= 2) {
17225 Result = StoreSubVec(2, NumBytes - Stored - 2,
17226 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
17227 RemainingBytes -= 2;
17230 if (RemainingBytes)
17232 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
17236 case PPC::BI__builtin_vsx_xvsqrtsp:
17237 case PPC::BI__builtin_vsx_xvsqrtdp: {
17238 llvm::Type *ResultType = ConvertType(E->
getType());
17240 if (Builder.getIsFPConstrained()) {
17241 llvm::Function *F = CGM.getIntrinsic(
17242 Intrinsic::experimental_constrained_sqrt, ResultType);
17243 return Builder.CreateConstrainedFPCall(F,
X);
17246 return Builder.CreateCall(F,
X);
17250 case PPC::BI__builtin_altivec_vclzb:
17251 case PPC::BI__builtin_altivec_vclzh:
17252 case PPC::BI__builtin_altivec_vclzw:
17253 case PPC::BI__builtin_altivec_vclzd: {
17254 llvm::Type *ResultType = ConvertType(E->
getType());
17256 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
17257 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
17258 return Builder.CreateCall(F, {
X, Undef});
17260 case PPC::BI__builtin_altivec_vctzb:
17261 case PPC::BI__builtin_altivec_vctzh:
17262 case PPC::BI__builtin_altivec_vctzw:
17263 case PPC::BI__builtin_altivec_vctzd: {
17264 llvm::Type *ResultType = ConvertType(E->
getType());
17266 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
17267 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
17268 return Builder.CreateCall(F, {
X, Undef});
17270 case PPC::BI__builtin_altivec_vinsd:
17271 case PPC::BI__builtin_altivec_vinsw:
17272 case PPC::BI__builtin_altivec_vinsd_elt:
17273 case PPC::BI__builtin_altivec_vinsw_elt: {
17274 llvm::Type *ResultType = ConvertType(E->
getType());
17279 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17280 BuiltinID == PPC::BI__builtin_altivec_vinsd);
17282 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17283 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
17286 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17288 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
17292 int ValidMaxValue = 0;
17294 ValidMaxValue = (Is32bit) ? 12 : 8;
17296 ValidMaxValue = (Is32bit) ? 3 : 1;
17299 int64_t ConstArg = ArgCI->getSExtValue();
17302 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
17303 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
17304 RangeErrMsg +=
" is outside of the valid range [0, ";
17305 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
17308 if (ConstArg < 0 || ConstArg > ValidMaxValue)
17312 if (!IsUnaligned) {
17313 ConstArg *= Is32bit ? 4 : 8;
17315 if (getTarget().isLittleEndian())
17316 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
17319 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
17320 Op2 = ConstantInt::getSigned(Int32Ty, ConstArg);
17324 ? Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4))
17325 : Builder.CreateBitCast(Op0,
17326 llvm::FixedVectorType::get(Int64Ty, 2));
17327 return Builder.CreateBitCast(
17328 Builder.CreateCall(CGM.getIntrinsic(
ID), {Op0, Op1, Op2}), ResultType);
17330 case PPC::BI__builtin_altivec_vpopcntb:
17331 case PPC::BI__builtin_altivec_vpopcnth:
17332 case PPC::BI__builtin_altivec_vpopcntw:
17333 case PPC::BI__builtin_altivec_vpopcntd: {
17334 llvm::Type *ResultType = ConvertType(E->
getType());
17336 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
17337 return Builder.CreateCall(F,
X);
17339 case PPC::BI__builtin_altivec_vadduqm:
17340 case PPC::BI__builtin_altivec_vsubuqm: {
17343 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
17344 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
17345 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
17346 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
17347 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
17349 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
17351 case PPC::BI__builtin_altivec_vaddcuq_c:
17352 case PPC::BI__builtin_altivec_vsubcuq_c: {
17356 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17357 llvm::IntegerType::get(getLLVMContext(), 128), 1);
17358 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
17359 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
17360 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
17361 ? Intrinsic::ppc_altivec_vaddcuq
17362 : Intrinsic::ppc_altivec_vsubcuq;
17363 return Builder.CreateCall(CGM.getIntrinsic(
ID), Ops,
"");
17365 case PPC::BI__builtin_altivec_vaddeuqm_c:
17366 case PPC::BI__builtin_altivec_vaddecuq_c:
17367 case PPC::BI__builtin_altivec_vsubeuqm_c:
17368 case PPC::BI__builtin_altivec_vsubecuq_c: {
17373 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17374 llvm::IntegerType::get(getLLVMContext(), 128), 1);
17375 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
17376 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
17377 Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty));
17378 switch (BuiltinID) {
17380 llvm_unreachable(
"Unsupported intrinsic!");
17381 case PPC::BI__builtin_altivec_vaddeuqm_c:
17382 ID = Intrinsic::ppc_altivec_vaddeuqm;
17384 case PPC::BI__builtin_altivec_vaddecuq_c:
17385 ID = Intrinsic::ppc_altivec_vaddecuq;
17387 case PPC::BI__builtin_altivec_vsubeuqm_c:
17388 ID = Intrinsic::ppc_altivec_vsubeuqm;
17390 case PPC::BI__builtin_altivec_vsubecuq_c:
17391 ID = Intrinsic::ppc_altivec_vsubecuq;
17394 return Builder.CreateCall(CGM.getIntrinsic(
ID), Ops,
"");
17396 case PPC::BI__builtin_ppc_rldimi:
17397 case PPC::BI__builtin_ppc_rlwimi: {
17404 if (BuiltinID == PPC::BI__builtin_ppc_rldimi &&
17405 !getTarget().getTriple().isPPC64()) {
17407 Op2 = Builder.CreateZExt(Op2, Int64Ty);
17408 Value *
Shift = Builder.CreateCall(F, {Op0, Op0, Op2});
17409 return Builder.CreateOr(Builder.CreateAnd(
Shift, Op3),
17410 Builder.CreateAnd(Op1, Builder.CreateNot(Op3)));
17412 return Builder.CreateCall(
17413 CGM.getIntrinsic(BuiltinID == PPC::BI__builtin_ppc_rldimi
17414 ? Intrinsic::ppc_rldimi
17415 : Intrinsic::ppc_rlwimi),
17416 {Op0, Op1, Op2, Op3});
17418 case PPC::BI__builtin_ppc_rlwnm: {
17422 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_rlwnm),
17425 case PPC::BI__builtin_ppc_poppar4:
17426 case PPC::BI__builtin_ppc_poppar8: {
17428 llvm::Type *ArgType = Op0->
getType();
17429 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
17430 Value *Tmp = Builder.CreateCall(F, Op0);
17432 llvm::Type *ResultType = ConvertType(E->
getType());
17433 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
17434 if (Result->getType() != ResultType)
17435 Result = Builder.CreateIntCast(Result, ResultType,
true,
17439 case PPC::BI__builtin_ppc_cmpb: {
17442 if (getTarget().getTriple().isPPC64()) {
17444 CGM.getIntrinsic(Intrinsic::ppc_cmpb, {Int64Ty, Int64Ty, Int64Ty});
17445 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
17462 CGM.getIntrinsic(Intrinsic::ppc_cmpb, {Int32Ty, Int32Ty, Int32Ty});
17463 Value *ArgOneLo = Builder.CreateTrunc(Op0, Int32Ty);
17464 Value *ArgTwoLo = Builder.CreateTrunc(Op1, Int32Ty);
17465 Constant *ShiftAmt = ConstantInt::get(Int64Ty, 32);
17467 Builder.CreateTrunc(Builder.CreateLShr(Op0, ShiftAmt), Int32Ty);
17469 Builder.CreateTrunc(Builder.CreateLShr(Op1, ShiftAmt), Int32Ty);
17470 Value *ResLo = Builder.CreateZExt(
17471 Builder.CreateCall(F, {ArgOneLo, ArgTwoLo},
"cmpb"), Int64Ty);
17472 Value *ResHiShift = Builder.CreateZExt(
17473 Builder.CreateCall(F, {ArgOneHi, ArgTwoHi},
"cmpb"), Int64Ty);
17474 Value *ResHi = Builder.CreateShl(ResHiShift, ShiftAmt);
17475 return Builder.CreateOr(ResLo, ResHi);
17478 case PPC::BI__builtin_vsx_xvcpsgnsp:
17479 case PPC::BI__builtin_vsx_xvcpsgndp: {
17480 llvm::Type *ResultType = ConvertType(E->
getType());
17484 llvm::Function *F = CGM.getIntrinsic(
ID, ResultType);
17485 return Builder.CreateCall(F, {
X, Y});
17488 case PPC::BI__builtin_vsx_xvrspip:
17489 case PPC::BI__builtin_vsx_xvrdpip:
17490 case PPC::BI__builtin_vsx_xvrdpim:
17491 case PPC::BI__builtin_vsx_xvrspim:
17492 case PPC::BI__builtin_vsx_xvrdpi:
17493 case PPC::BI__builtin_vsx_xvrspi:
17494 case PPC::BI__builtin_vsx_xvrdpic:
17495 case PPC::BI__builtin_vsx_xvrspic:
17496 case PPC::BI__builtin_vsx_xvrdpiz:
17497 case PPC::BI__builtin_vsx_xvrspiz: {
17498 llvm::Type *ResultType = ConvertType(E->
getType());
17500 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
17501 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
17502 ID = Builder.getIsFPConstrained()
17503 ? Intrinsic::experimental_constrained_floor
17505 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
17506 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
17507 ID = Builder.getIsFPConstrained()
17508 ? Intrinsic::experimental_constrained_round
17510 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
17511 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
17512 ID = Builder.getIsFPConstrained()
17513 ? Intrinsic::experimental_constrained_rint
17515 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
17516 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
17517 ID = Builder.getIsFPConstrained()
17518 ? Intrinsic::experimental_constrained_ceil
17520 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
17521 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
17522 ID = Builder.getIsFPConstrained()
17523 ? Intrinsic::experimental_constrained_trunc
17525 llvm::Function *F = CGM.getIntrinsic(
ID, ResultType);
17526 return Builder.getIsFPConstrained() ? Builder.CreateConstrainedFPCall(F,
X)
17527 : Builder.CreateCall(F,
X);
17531 case PPC::BI__builtin_vsx_xvabsdp:
17532 case PPC::BI__builtin_vsx_xvabssp: {
17533 llvm::Type *ResultType = ConvertType(E->
getType());
17536 return Builder.CreateCall(F,
X);
17540 case PPC::BI__builtin_ppc_recipdivf:
17541 case PPC::BI__builtin_ppc_recipdivd:
17542 case PPC::BI__builtin_ppc_rsqrtf:
17543 case PPC::BI__builtin_ppc_rsqrtd: {
17544 FastMathFlags FMF = Builder.getFastMathFlags();
17545 Builder.getFastMathFlags().setFast();
17546 llvm::Type *ResultType = ConvertType(E->
getType());
17549 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
17550 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
17552 Value *FDiv = Builder.CreateFDiv(
X, Y,
"recipdiv");
17553 Builder.getFastMathFlags() &= (FMF);
17556 auto *One = ConstantFP::get(ResultType, 1.0);
17558 Value *FDiv = Builder.CreateFDiv(One, Builder.CreateCall(F,
X),
"rsqrt");
17559 Builder.getFastMathFlags() &= (FMF);
17562 case PPC::BI__builtin_ppc_alignx: {
17565 ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
17566 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
17567 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
17568 llvm::Value::MaximumAlignment);
17570 emitAlignmentAssumption(Op1, E->
getArg(1),
17572 AlignmentCI,
nullptr);
17575 case PPC::BI__builtin_ppc_rdlam: {
17579 llvm::Type *Ty = Op0->
getType();
17580 Value *ShiftAmt = Builder.CreateIntCast(Op1, Ty,
false);
17581 Function *F = CGM.getIntrinsic(Intrinsic::fshl, Ty);
17582 Value *Rotate = Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
17583 return Builder.CreateAnd(Rotate, Op2);
17585 case PPC::BI__builtin_ppc_load2r: {
17586 Function *F = CGM.getIntrinsic(Intrinsic::ppc_load2r);
17588 Value *LoadIntrinsic = Builder.CreateCall(F, {Op0});
17589 return Builder.CreateTrunc(LoadIntrinsic, Int16Ty);
17592 case PPC::BI__builtin_ppc_fnmsub:
17593 case PPC::BI__builtin_ppc_fnmsubs:
17594 case PPC::BI__builtin_vsx_xvmaddadp:
17595 case PPC::BI__builtin_vsx_xvmaddasp:
17596 case PPC::BI__builtin_vsx_xvnmaddadp:
17597 case PPC::BI__builtin_vsx_xvnmaddasp:
17598 case PPC::BI__builtin_vsx_xvmsubadp:
17599 case PPC::BI__builtin_vsx_xvmsubasp:
17600 case PPC::BI__builtin_vsx_xvnmsubadp:
17601 case PPC::BI__builtin_vsx_xvnmsubasp: {
17602 llvm::Type *ResultType = ConvertType(E->
getType());
17607 if (Builder.getIsFPConstrained())
17608 F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17611 switch (BuiltinID) {
17612 case PPC::BI__builtin_vsx_xvmaddadp:
17613 case PPC::BI__builtin_vsx_xvmaddasp:
17614 if (Builder.getIsFPConstrained())
17615 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
17617 return Builder.CreateCall(F, {
X, Y, Z});
17618 case PPC::BI__builtin_vsx_xvnmaddadp:
17619 case PPC::BI__builtin_vsx_xvnmaddasp:
17620 if (Builder.getIsFPConstrained())
17621 return Builder.CreateFNeg(
17622 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
17624 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}),
"neg");
17625 case PPC::BI__builtin_vsx_xvmsubadp:
17626 case PPC::BI__builtin_vsx_xvmsubasp:
17627 if (Builder.getIsFPConstrained())
17628 return Builder.CreateConstrainedFPCall(
17629 F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
17631 return Builder.CreateCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
17632 case PPC::BI__builtin_ppc_fnmsub:
17633 case PPC::BI__builtin_ppc_fnmsubs:
17634 case PPC::BI__builtin_vsx_xvnmsubadp:
17635 case PPC::BI__builtin_vsx_xvnmsubasp:
17636 if (Builder.getIsFPConstrained())
17637 return Builder.CreateFNeg(
17638 Builder.CreateConstrainedFPCall(
17639 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
17642 return Builder.CreateCall(
17643 CGM.getIntrinsic(Intrinsic::ppc_fnmsub, ResultType), {X, Y, Z});
17645 llvm_unreachable(
"Unknown FMA operation");
17649 case PPC::BI__builtin_vsx_insertword: {
17653 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxinsertw);
17657 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17659 "Third arg to xxinsertw intrinsic must be constant integer");
17668 std::swap(Op0, Op1);
17672 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int64Ty, 2));
17674 if (getTarget().isLittleEndian()) {
17676 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
17677 Op0 = Builder.CreateShuffleVector(Op0, Op0,
ArrayRef<int>{1, 0});
17680 Index = MaxIndex - Index;
17684 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4));
17685 Op2 = ConstantInt::getSigned(Int32Ty, Index);
17686 return Builder.CreateCall(F, {Op0, Op1, Op2});
17689 case PPC::BI__builtin_vsx_extractuword: {
17692 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
17695 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
17699 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
17701 "Second Arg to xxextractuw intrinsic must be a constant integer!");
17705 if (getTarget().isLittleEndian()) {
17707 Index = MaxIndex - Index;
17708 Op1 = ConstantInt::getSigned(Int32Ty, Index);
17711 Value *Call = Builder.CreateCall(F, {Op0, Op1});
17713 Value *ShuffleCall =
17714 Builder.CreateShuffleVector(Call, Call,
ArrayRef<int>{1, 0});
17715 return ShuffleCall;
17717 Op1 = ConstantInt::getSigned(Int32Ty, Index);
17718 return Builder.CreateCall(F, {Op0, Op1});
17722 case PPC::BI__builtin_vsx_xxpermdi: {
17726 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17727 assert(ArgCI &&
"Third arg must be constant integer!");
17729 unsigned Index = ArgCI->getZExtValue();
17730 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
17731 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int64Ty, 2));
17736 int ElemIdx0 = (Index & 2) >> 1;
17737 int ElemIdx1 = 2 + (Index & 1);
17739 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
17740 Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17742 auto RetTy = ConvertType(BIRetType);
17743 return Builder.CreateBitCast(ShuffleCall, RetTy);
17746 case PPC::BI__builtin_vsx_xxsldwi: {
17750 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17751 assert(ArgCI &&
"Third argument must be a compile time constant");
17752 unsigned Index = ArgCI->getZExtValue() & 0x3;
17753 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4));
17754 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int32Ty, 4));
17761 if (getTarget().isLittleEndian()) {
17765 ElemIdx0 = (8 - Index) % 8;
17766 ElemIdx1 = (9 - Index) % 8;
17767 ElemIdx2 = (10 - Index) % 8;
17768 ElemIdx3 = (11 - Index) % 8;
17772 ElemIdx1 = Index + 1;
17773 ElemIdx2 = Index + 2;
17774 ElemIdx3 = Index + 3;
17777 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
17778 Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17780 auto RetTy = ConvertType(BIRetType);
17781 return Builder.CreateBitCast(ShuffleCall, RetTy);
17784 case PPC::BI__builtin_pack_vector_int128: {
17787 bool isLittleEndian = getTarget().isLittleEndian();
17788 Value *PoisonValue =
17789 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
17790 Value *Res = Builder.CreateInsertElement(
17791 PoisonValue, Op0, (
uint64_t)(isLittleEndian ? 1 : 0));
17792 Res = Builder.CreateInsertElement(Res, Op1,
17793 (
uint64_t)(isLittleEndian ? 0 : 1));
17794 return Builder.CreateBitCast(Res, ConvertType(E->
getType()));
17797 case PPC::BI__builtin_unpack_vector_int128: {
17800 ConstantInt *Index = cast<ConstantInt>(Op1);
17801 Value *Unpacked = Builder.CreateBitCast(
17802 Op0, llvm::FixedVectorType::get(ConvertType(E->
getType()), 2));
17804 if (getTarget().isLittleEndian())
17806 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
17808 return Builder.CreateExtractElement(Unpacked, Index);
17811 case PPC::BI__builtin_ppc_sthcx: {
17812 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_sthcx);
17814 Value *Op1 = Builder.CreateSExt(EmitScalarExpr(E->
getArg(1)), Int32Ty);
17815 return Builder.CreateCall(F, {Op0, Op1});
17824 #define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
17825 case PPC::BI__builtin_##Name:
17826 #include "clang/Basic/BuiltinsPPC.def"
17829 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
17832 EmitArrayToPointerDecay(E->
getArg(i)).emitRawPointer(*
this));
17834 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
17839 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
17840 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
17841 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
17842 unsigned NumVecs = 2;
17843 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
17844 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
17846 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
17848 llvm::Function *F = CGM.getIntrinsic(Intrinsic);
17850 Value *Vec = Builder.CreateLoad(Addr);
17851 Value *Call = Builder.CreateCall(F, {Vec});
17852 llvm::Type *VTy = llvm::FixedVectorType::get(Int8Ty, 16);
17853 Value *Ptr = Ops[0];
17854 for (
unsigned i=0; i<NumVecs; i++) {
17855 Value *Vec = Builder.CreateExtractValue(Call, i);
17856 llvm::ConstantInt* Index = llvm::ConstantInt::get(IntTy, i);
17857 Value *GEP = Builder.CreateInBoundsGEP(VTy, Ptr, Index);
17858 Builder.CreateAlignedStore(Vec, GEP, MaybeAlign(16));
17862 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
17863 BuiltinID == PPC::BI__builtin_mma_build_acc) {
17870 if (getTarget().isLittleEndian())
17871 std::reverse(Ops.begin() + 1, Ops.end());
17874 switch (BuiltinID) {
17875 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
17876 case PPC::BI__builtin_##Name: \
17877 ID = Intrinsic::ppc_##Intr; \
17878 Accumulate = Acc; \
17880 #include "clang/Basic/BuiltinsPPC.def"
17882 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17883 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
17884 BuiltinID == PPC::BI__builtin_mma_lxvp ||
17885 BuiltinID == PPC::BI__builtin_mma_stxvp) {
17886 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17887 BuiltinID == PPC::BI__builtin_mma_lxvp) {
17888 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]);
17890 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]);
17893 llvm::Function *F = CGM.getIntrinsic(
ID);
17894 return Builder.CreateCall(F, Ops,
"");
17899 Value *Acc = Builder.CreateLoad(Addr);
17900 CallOps.push_back(Acc);
17902 for (
unsigned i=1; i<Ops.size(); i++)
17903 CallOps.push_back(Ops[i]);
17904 llvm::Function *F = CGM.getIntrinsic(
ID);
17905 Value *Call = Builder.CreateCall(F, CallOps);
17906 return Builder.CreateAlignedStore(Call, Ops[0], MaybeAlign(64));
17909 case PPC::BI__builtin_ppc_compare_and_swap:
17910 case PPC::BI__builtin_ppc_compare_and_swaplp: {
17913 Value *OldVal = Builder.CreateLoad(OldValAddr);
17915 LValue LV = MakeAddrLValue(Addr, AtomicTy);
17917 auto Pair = EmitAtomicCompareExchange(
17919 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
17927 Value *LoadedVal = Pair.first.getScalarVal();
17928 Builder.CreateStore(LoadedVal, OldValAddr);
17929 return Builder.CreateZExt(Pair.second, Builder.getInt32Ty());
17931 case PPC::BI__builtin_ppc_fetch_and_add:
17932 case PPC::BI__builtin_ppc_fetch_and_addlp: {
17934 llvm::AtomicOrdering::Monotonic);
17936 case PPC::BI__builtin_ppc_fetch_and_and:
17937 case PPC::BI__builtin_ppc_fetch_and_andlp: {
17939 llvm::AtomicOrdering::Monotonic);
17942 case PPC::BI__builtin_ppc_fetch_and_or:
17943 case PPC::BI__builtin_ppc_fetch_and_orlp: {
17945 llvm::AtomicOrdering::Monotonic);
17947 case PPC::BI__builtin_ppc_fetch_and_swap:
17948 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
17950 llvm::AtomicOrdering::Monotonic);
17952 case PPC::BI__builtin_ppc_ldarx:
17953 case PPC::BI__builtin_ppc_lwarx:
17954 case PPC::BI__builtin_ppc_lharx:
17955 case PPC::BI__builtin_ppc_lbarx:
17957 case PPC::BI__builtin_ppc_mfspr: {
17959 llvm::Type *RetType = CGM.getDataLayout().getTypeSizeInBits(VoidPtrTy) == 32
17962 Function *F = CGM.getIntrinsic(Intrinsic::ppc_mfspr, RetType);
17963 return Builder.CreateCall(F, {Op0});
17965 case PPC::BI__builtin_ppc_mtspr: {
17968 llvm::Type *RetType = CGM.getDataLayout().getTypeSizeInBits(VoidPtrTy) == 32
17971 Function *F = CGM.getIntrinsic(Intrinsic::ppc_mtspr, RetType);
17972 return Builder.CreateCall(F, {Op0, Op1});
17974 case PPC::BI__builtin_ppc_popcntb: {
17976 llvm::Type *ArgType = ArgValue->
getType();
17977 Function *F = CGM.getIntrinsic(Intrinsic::ppc_popcntb, {ArgType, ArgType});
17978 return Builder.CreateCall(F, {ArgValue},
"popcntb");
17980 case PPC::BI__builtin_ppc_mtfsf: {
17985 Value *
Cast = Builder.CreateUIToFP(Op1, DoubleTy);
17986 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_mtfsf);
17987 return Builder.CreateCall(F, {Op0,
Cast},
"");
17990 case PPC::BI__builtin_ppc_swdiv_nochk:
17991 case PPC::BI__builtin_ppc_swdivs_nochk: {
17994 FastMathFlags FMF = Builder.getFastMathFlags();
17995 Builder.getFastMathFlags().setFast();
17996 Value *FDiv = Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
17997 Builder.getFastMathFlags() &= (FMF);
18000 case PPC::BI__builtin_ppc_fric:
18003 Intrinsic::experimental_constrained_rint))
18005 case PPC::BI__builtin_ppc_frim:
18006 case PPC::BI__builtin_ppc_frims:
18009 Intrinsic::experimental_constrained_floor))
18011 case PPC::BI__builtin_ppc_frin:
18012 case PPC::BI__builtin_ppc_frins:
18015 Intrinsic::experimental_constrained_round))
18017 case PPC::BI__builtin_ppc_frip:
18018 case PPC::BI__builtin_ppc_frips:
18021 Intrinsic::experimental_constrained_ceil))
18023 case PPC::BI__builtin_ppc_friz:
18024 case PPC::BI__builtin_ppc_frizs:
18027 Intrinsic::experimental_constrained_trunc))
18029 case PPC::BI__builtin_ppc_fsqrt:
18030 case PPC::BI__builtin_ppc_fsqrts:
18033 Intrinsic::experimental_constrained_sqrt))
18035 case PPC::BI__builtin_ppc_test_data_class: {
18038 return Builder.CreateCall(
18039 CGM.getIntrinsic(Intrinsic::ppc_test_data_class, Op0->
getType()),
18040 {Op0, Op1},
"test_data_class");
18042 case PPC::BI__builtin_ppc_maxfe: {
18047 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfe),
18048 {Op0, Op1, Op2, Op3});
18050 case PPC::BI__builtin_ppc_maxfl: {
18055 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfl),
18056 {Op0, Op1, Op2, Op3});
18058 case PPC::BI__builtin_ppc_maxfs: {
18063 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfs),
18064 {Op0, Op1, Op2, Op3});
18066 case PPC::BI__builtin_ppc_minfe: {
18071 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfe),
18072 {Op0, Op1, Op2, Op3});
18074 case PPC::BI__builtin_ppc_minfl: {
18079 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfl),
18080 {Op0, Op1, Op2, Op3});
18082 case PPC::BI__builtin_ppc_minfs: {
18087 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfs),
18088 {Op0, Op1, Op2, Op3});
18090 case PPC::BI__builtin_ppc_swdiv:
18091 case PPC::BI__builtin_ppc_swdivs: {
18094 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
18096 case PPC::BI__builtin_ppc_set_fpscr_rn:
18097 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_setrnd),
18098 {EmitScalarExpr(E->getArg(0))});
18099 case PPC::BI__builtin_ppc_mffs:
18100 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_readflm));
18110 auto *Call = CGF.
Builder.CreateCall(F);
18112 Attribute::getWithDereferenceableBytes(Call->getContext(), 64));
18113 Call->addRetAttr(Attribute::getWithAlignment(Call->getContext(), Align(4)));
18117 auto *RetTy = cast<llvm::PointerType>(CGF.
ConvertType(BuiltinRetType));
18118 if (RetTy == Call->getType())
18125 auto *Call = CGF.
Builder.CreateCall(F);
18127 Attribute::getWithDereferenceableBytes(Call->getContext(), 256));
18128 Call->addRetAttr(Attribute::getWithAlignment(Call->getContext(), Align(8)));
18143 llvm::LoadInst *LD;
18147 if (Cov == CodeObjectVersionKind::COV_None) {
18148 StringRef Name =
"__oclc_ABI_version";
18149 auto *ABIVersionC = CGF.
CGM.
getModule().getNamedGlobal(Name);
18151 ABIVersionC =
new llvm::GlobalVariable(
18153 llvm::GlobalValue::ExternalLinkage,
nullptr, Name,
nullptr,
18154 llvm::GlobalVariable::NotThreadLocal,
18165 llvm::ConstantInt::get(CGF.
Int32Ty, CodeObjectVersionKind::COV_5));
18169 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
18173 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
18175 auto Result = CGF.
Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
18179 Value *GEP =
nullptr;
18180 if (Cov >= CodeObjectVersionKind::COV_5) {
18182 GEP = CGF.
Builder.CreateConstGEP1_32(
18183 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
18186 GEP = CGF.
Builder.CreateConstGEP1_32(
18187 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
18194 llvm::MDNode *RNode = MDHelper.createRange(
APInt(16, 1),
18196 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
18197 LD->setMetadata(llvm::LLVMContext::MD_noundef,
18199 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18206 const unsigned XOffset = 12;
18207 auto *DP = EmitAMDGPUDispatchPtr(CGF);
18209 auto *
Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
18213 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18226 llvm::AtomicOrdering &AO,
18228 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
18231 assert(llvm::isValidAtomicOrderingCABI(ord));
18232 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
18233 case llvm::AtomicOrderingCABI::acquire:
18234 case llvm::AtomicOrderingCABI::consume:
18235 AO = llvm::AtomicOrdering::Acquire;
18237 case llvm::AtomicOrderingCABI::release:
18238 AO = llvm::AtomicOrdering::Release;
18240 case llvm::AtomicOrderingCABI::acq_rel:
18241 AO = llvm::AtomicOrdering::AcquireRelease;
18243 case llvm::AtomicOrderingCABI::seq_cst:
18244 AO = llvm::AtomicOrdering::SequentiallyConsistent;
18246 case llvm::AtomicOrderingCABI::relaxed:
18247 AO = llvm::AtomicOrdering::Monotonic;
18252 llvm::getConstantStringInfo(
Scope, scp);
18253 SSID = getLLVMContext().getOrInsertSyncScopeID(scp);
18259 llvm::Value *Arg =
nullptr;
18260 if ((ICEArguments & (1 << Idx)) == 0) {
18261 Arg = EmitScalarExpr(E->
getArg(Idx));
18265 std::optional<llvm::APSInt> Result =
18267 assert(Result &&
"Expected argument to be a constant");
18268 Arg = llvm::ConstantInt::get(getLLVMContext(), *Result);
18275 switch (elementCount) {
18277 return Intrinsic::dx_dot2;
18279 return Intrinsic::dx_dot3;
18281 return Intrinsic::dx_dot4;
18285 return Intrinsic::dx_sdot;
18288 return Intrinsic::dx_udot;
18293 if (!getLangOpts().
HLSL)
18296 switch (BuiltinID) {
18297 case Builtin::BI__builtin_hlsl_elementwise_all: {
18299 return Builder.CreateIntrinsic(
18300 llvm::Type::getInt1Ty(getLLVMContext()),
18304 case Builtin::BI__builtin_hlsl_elementwise_any: {
18306 return Builder.CreateIntrinsic(
18307 llvm::Type::getInt1Ty(getLLVMContext()),
18311 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
18317 bool IsUnsigned =
false;
18319 Ty = VecTy->getElementType();
18321 return Builder.CreateIntrinsic(
18323 IsUnsigned ? Intrinsic::dx_uclamp : Intrinsic::dx_clamp,
18326 case Builtin::BI__builtin_hlsl_dot: {
18329 llvm::Type *T0 = Op0->
getType();
18330 llvm::Type *T1 = Op1->
getType();
18331 if (!T0->isVectorTy() && !T1->isVectorTy()) {
18332 if (T0->isFloatingPointTy())
18333 return Builder.CreateFMul(Op0, Op1,
"dx.dot");
18335 if (T0->isIntegerTy())
18336 return Builder.CreateMul(Op0, Op1,
"dx.dot");
18340 "Scalar dot product is only supported on ints and floats.");
18343 assert(T0->isVectorTy() && T1->isVectorTy() &&
18344 "Dot product of vector and scalar is not supported.");
18347 assert(T0->getScalarType() == T1->getScalarType() &&
18348 "Dot product of vectors need the same element types.");
18351 [[maybe_unused]]
auto *VecTy1 =
18355 "Dot product requires vectors to be of the same size.");
18357 return Builder.CreateIntrinsic(
18358 T0->getScalarType(),
18360 VecTy0->getNumElements()),
18363 case Builtin::BI__builtin_hlsl_lerp: {
18368 llvm_unreachable(
"lerp operand must have a float representation");
18369 return Builder.CreateIntrinsic(
18370 X->getType(), CGM.getHLSLRuntime().getLerpIntrinsic(),
18373 case Builtin::BI__builtin_hlsl_elementwise_frac: {
18376 llvm_unreachable(
"frac operand must have a float representation");
18377 return Builder.CreateIntrinsic(
18378 Op0->
getType(), Intrinsic::dx_frac,
18381 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
18383 llvm::Type *Xty = Op0->
getType();
18384 llvm::Type *retType = llvm::Type::getInt1Ty(this->getLLVMContext());
18385 if (Xty->isVectorTy()) {
18387 retType = llvm::VectorType::get(
18388 retType, ElementCount::getFixed(XVecTy->getNumElements()));
18391 llvm_unreachable(
"isinf operand must have a float representation");
18392 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
18395 case Builtin::BI__builtin_hlsl_mad: {
18400 return Builder.CreateIntrinsic(
18401 M->
getType(), Intrinsic::fmuladd,
18405 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
18406 return Builder.CreateIntrinsic(
18407 M->
getType(), Intrinsic::dx_imad,
18410 Value *
Mul = Builder.CreateNSWMul(M, A);
18411 return Builder.CreateNSWAdd(
Mul, B);
18414 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
18415 return Builder.CreateIntrinsic(
18416 M->
getType(), Intrinsic::dx_umad,
18419 Value *
Mul = Builder.CreateNUWMul(M, A);
18420 return Builder.CreateNUWAdd(
Mul, B);
18422 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
18425 llvm_unreachable(
"rcp operand must have a float representation");
18426 llvm::Type *Ty = Op0->
getType();
18427 llvm::Type *EltTy = Ty->getScalarType();
18430 ? ConstantVector::getSplat(
18431 ElementCount::getFixed(
18432 dyn_cast<FixedVectorType>(Ty)->getNumElements()),
18433 ConstantFP::get(EltTy, 1.0))
18434 : ConstantFP::get(EltTy, 1.0);
18435 return Builder.CreateFDiv(One, Op0,
"hlsl.rcp");
18437 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
18440 llvm_unreachable(
"rsqrt operand must have a float representation");
18441 return Builder.CreateIntrinsic(
18442 Op0->
getType(), Intrinsic::dx_rsqrt,
18445 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
18446 auto *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction(
18447 llvm::FunctionType::get(IntTy, {},
false),
"__hlsl_wave_get_lane_index",
18449 if (getTarget().getTriple().isSPIRVLogical())
18450 CI = dyn_cast<CallInst>(addControlledConvergenceToken(CI));
18459 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
18461 switch (BuiltinID) {
18462 case AMDGPU::BI__builtin_amdgcn_div_scale:
18463 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
18469 llvm::Value *
X = EmitScalarExpr(E->
getArg(0));
18470 llvm::Value *Y = EmitScalarExpr(E->
getArg(1));
18471 llvm::Value *Z = EmitScalarExpr(E->
getArg(2));
18473 llvm::Function *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale,
18476 llvm::Value *Tmp = Builder.CreateCall(Callee, {
X, Y, Z});
18478 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
18479 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
18483 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
18484 Builder.CreateStore(FlagExt, FlagOutPtr);
18487 case AMDGPU::BI__builtin_amdgcn_div_fmas:
18488 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
18489 llvm::Value *Src0 = EmitScalarExpr(E->
getArg(0));
18490 llvm::Value *Src1 = EmitScalarExpr(E->
getArg(1));
18491 llvm::Value *Src2 = EmitScalarExpr(E->
getArg(2));
18492 llvm::Value *Src3 = EmitScalarExpr(E->
getArg(3));
18494 llvm::Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas,
18496 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
18497 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
18500 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
18502 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
18504 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
18505 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
18509 unsigned ICEArguments = 0;
18511 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
18513 for (
unsigned I = 0; I != E->
getNumArgs(); ++I) {
18514 Args.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, I, E));
18516 assert(Args.size() == 5 || Args.size() == 6);
18517 if (Args.size() == 5)
18518 Args.insert(Args.begin(), llvm::PoisonValue::get(Args[0]->getType()));
18520 CGM.getIntrinsic(Intrinsic::amdgcn_update_dpp, Args[0]->getType());
18521 return Builder.CreateCall(F, Args);
18523 case AMDGPU::BI__builtin_amdgcn_div_fixup:
18524 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
18525 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
18527 case AMDGPU::BI__builtin_amdgcn_trig_preop:
18528 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
18530 case AMDGPU::BI__builtin_amdgcn_rcp:
18531 case AMDGPU::BI__builtin_amdgcn_rcpf:
18532 case AMDGPU::BI__builtin_amdgcn_rcph:
18534 case AMDGPU::BI__builtin_amdgcn_sqrt:
18535 case AMDGPU::BI__builtin_amdgcn_sqrtf:
18536 case AMDGPU::BI__builtin_amdgcn_sqrth:
18538 case AMDGPU::BI__builtin_amdgcn_rsq:
18539 case AMDGPU::BI__builtin_amdgcn_rsqf:
18540 case AMDGPU::BI__builtin_amdgcn_rsqh:
18542 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
18543 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
18545 case AMDGPU::BI__builtin_amdgcn_sinf:
18546 case AMDGPU::BI__builtin_amdgcn_sinh:
18548 case AMDGPU::BI__builtin_amdgcn_cosf:
18549 case AMDGPU::BI__builtin_amdgcn_cosh:
18551 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
18552 return EmitAMDGPUDispatchPtr(*
this, E);
18553 case AMDGPU::BI__builtin_amdgcn_logf:
18555 case AMDGPU::BI__builtin_amdgcn_exp2f:
18557 case AMDGPU::BI__builtin_amdgcn_log_clampf:
18559 case AMDGPU::BI__builtin_amdgcn_ldexp:
18560 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
18561 llvm::Value *Src0 = EmitScalarExpr(E->
getArg(0));
18562 llvm::Value *Src1 = EmitScalarExpr(E->
getArg(1));
18563 llvm::Function *F =
18565 return Builder.CreateCall(F, {Src0, Src1});
18567 case AMDGPU::BI__builtin_amdgcn_ldexph: {
18570 llvm::Value *Src0 = EmitScalarExpr(E->
getArg(0));
18571 llvm::Value *Src1 = EmitScalarExpr(E->
getArg(1));
18572 llvm::Function *F =
18574 return Builder.CreateCall(F, {Src0, Builder.CreateTrunc(Src1, Int16Ty)});
18576 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
18577 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
18578 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
18580 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
18581 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
18583 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
18584 { Builder.getInt32Ty(), Src0->
getType() });
18585 return Builder.CreateCall(F, Src0);
18587 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
18589 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
18590 { Builder.getInt16Ty(), Src0->
getType() });
18591 return Builder.CreateCall(F, Src0);
18593 case AMDGPU::BI__builtin_amdgcn_fract:
18594 case AMDGPU::BI__builtin_amdgcn_fractf:
18595 case AMDGPU::BI__builtin_amdgcn_fracth:
18597 case AMDGPU::BI__builtin_amdgcn_lerp:
18599 case AMDGPU::BI__builtin_amdgcn_ubfe:
18601 case AMDGPU::BI__builtin_amdgcn_sbfe:
18603 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
18604 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
18605 llvm::Type *ResultType = ConvertType(E->
getType());
18606 llvm::Value *Src = EmitScalarExpr(E->
getArg(0));
18607 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_ballot, { ResultType });
18608 return Builder.CreateCall(F, { Src });
18610 case AMDGPU::BI__builtin_amdgcn_uicmp:
18611 case AMDGPU::BI__builtin_amdgcn_uicmpl:
18612 case AMDGPU::BI__builtin_amdgcn_sicmp:
18613 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
18614 llvm::Value *Src0 = EmitScalarExpr(E->
getArg(0));
18615 llvm::Value *Src1 = EmitScalarExpr(E->
getArg(1));
18616 llvm::Value *Src2 = EmitScalarExpr(E->
getArg(2));
18619 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_icmp,
18620 { Builder.getInt64Ty(), Src0->getType() });
18621 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18623 case AMDGPU::BI__builtin_amdgcn_fcmp:
18624 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
18625 llvm::Value *Src0 = EmitScalarExpr(E->
getArg(0));
18626 llvm::Value *Src1 = EmitScalarExpr(E->
getArg(1));
18627 llvm::Value *Src2 = EmitScalarExpr(E->
getArg(2));
18630 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_fcmp,
18631 { Builder.getInt64Ty(), Src0->getType() });
18632 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18634 case AMDGPU::BI__builtin_amdgcn_class:
18635 case AMDGPU::BI__builtin_amdgcn_classf:
18636 case AMDGPU::BI__builtin_amdgcn_classh:
18638 case AMDGPU::BI__builtin_amdgcn_fmed3f:
18639 case AMDGPU::BI__builtin_amdgcn_fmed3h:
18641 case AMDGPU::BI__builtin_amdgcn_ds_append:
18642 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
18643 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
18644 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
18647 return Builder.CreateCall(F, { Src0, Builder.getFalse() });
18649 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
18650 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
18651 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
18653 switch (BuiltinID) {
18654 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
18655 Intrin = Intrinsic::amdgcn_ds_fadd;
18657 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
18658 Intrin = Intrinsic::amdgcn_ds_fmin;
18660 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
18661 Intrin = Intrinsic::amdgcn_ds_fmax;
18664 llvm::Value *Src0 = EmitScalarExpr(E->
getArg(0));
18665 llvm::Value *Src1 = EmitScalarExpr(E->
getArg(1));
18666 llvm::Value *Src2 = EmitScalarExpr(E->
getArg(2));
18667 llvm::Value *Src3 = EmitScalarExpr(E->
getArg(3));
18668 llvm::Value *Src4 = EmitScalarExpr(E->
getArg(4));
18669 llvm::Function *F = CGM.getIntrinsic(Intrin, { Src1->getType() });
18670 llvm::FunctionType *FTy = F->getFunctionType();
18671 llvm::Type *PTy = FTy->getParamType(0);
18672 Src0 = Builder.CreatePointerBitCastOrAddrSpaceCast(Src0, PTy);
18673 return Builder.CreateCall(F, { Src0, Src1, Src2, Src3, Src4 });
18675 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18676 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18677 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18678 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18679 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18680 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18681 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18682 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18683 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18684 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: {
18686 llvm::Type *ArgTy = llvm::Type::getDoubleTy(getLLVMContext());
18687 switch (BuiltinID) {
18688 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18689 ArgTy = llvm::Type::getFloatTy(getLLVMContext());
18690 IID = Intrinsic::amdgcn_global_atomic_fadd;
18692 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18693 ArgTy = llvm::FixedVectorType::get(
18694 llvm::Type::getHalfTy(getLLVMContext()), 2);
18695 IID = Intrinsic::amdgcn_global_atomic_fadd;
18697 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18698 IID = Intrinsic::amdgcn_global_atomic_fadd;
18700 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18701 IID = Intrinsic::amdgcn_global_atomic_fmin;
18703 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18704 IID = Intrinsic::amdgcn_global_atomic_fmax;
18706 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18707 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18709 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18710 IID = Intrinsic::amdgcn_flat_atomic_fmin;
18712 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18713 IID = Intrinsic::amdgcn_flat_atomic_fmax;
18715 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18716 ArgTy = llvm::Type::getFloatTy(getLLVMContext());
18717 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18719 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
18720 ArgTy = llvm::FixedVectorType::get(
18721 llvm::Type::getHalfTy(getLLVMContext()), 2);
18722 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18725 llvm::Value *Addr = EmitScalarExpr(E->
getArg(0));
18726 llvm::Value *Val = EmitScalarExpr(E->
getArg(1));
18727 llvm::Function *F =
18728 CGM.getIntrinsic(IID, {ArgTy, Addr->getType(), Val->getType()});
18729 return Builder.CreateCall(F, {Addr, Val});
18731 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18732 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: {
18734 switch (BuiltinID) {
18735 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18736 IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
18738 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
18739 IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
18742 llvm::Value *Addr = EmitScalarExpr(E->
getArg(0));
18743 llvm::Value *Val = EmitScalarExpr(E->
getArg(1));
18744 llvm::Function *F = CGM.getIntrinsic(IID, {Addr->getType()});
18745 return Builder.CreateCall(F, {Addr, Val});
18747 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
18748 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
18749 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: {
18752 switch (BuiltinID) {
18753 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
18754 ArgTy = llvm::Type::getFloatTy(getLLVMContext());
18755 IID = Intrinsic::amdgcn_ds_fadd;
18757 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
18758 ArgTy = llvm::Type::getDoubleTy(getLLVMContext());
18759 IID = Intrinsic::amdgcn_ds_fadd;
18761 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
18762 ArgTy = llvm::FixedVectorType::get(
18763 llvm::Type::getHalfTy(getLLVMContext()), 2);
18764 IID = Intrinsic::amdgcn_ds_fadd;
18767 llvm::Value *Addr = EmitScalarExpr(E->
getArg(0));
18768 llvm::Value *Val = EmitScalarExpr(E->
getArg(1));
18769 llvm::Constant *ZeroI32 = llvm::ConstantInt::getIntegerValue(
18770 llvm::Type::getInt32Ty(getLLVMContext()),
APInt(32, 0,
true));
18771 llvm::Constant *ZeroI1 = llvm::ConstantInt::getIntegerValue(
18772 llvm::Type::getInt1Ty(getLLVMContext()),
APInt(1, 0));
18773 llvm::Function *F = CGM.getIntrinsic(IID, {ArgTy});
18774 return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1});
18776 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
18777 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
18778 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
18779 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: {
18782 switch (BuiltinID) {
18783 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
18784 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
18785 IID = Intrinsic::amdgcn_global_load_tr_b64;
18787 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
18788 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
18789 IID = Intrinsic::amdgcn_global_load_tr_b128;
18792 llvm::Type *LoadTy = ConvertType(E->
getType());
18793 llvm::Value *Addr = EmitScalarExpr(E->
getArg(0));
18794 llvm::Function *F = CGM.getIntrinsic(IID, {LoadTy});
18795 return Builder.CreateCall(F, {Addr});
18797 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
18798 Function *F = CGM.getIntrinsic(Intrinsic::get_fpenv,
18799 {llvm::Type::getInt64Ty(getLLVMContext())});
18800 return Builder.CreateCall(F);
18802 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
18803 Function *F = CGM.getIntrinsic(Intrinsic::set_fpenv,
18804 {llvm::Type::getInt64Ty(getLLVMContext())});
18805 llvm::Value *
Env = EmitScalarExpr(E->
getArg(0));
18806 return Builder.CreateCall(F, {
Env});
18808 case AMDGPU::BI__builtin_amdgcn_read_exec:
18810 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
18812 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
18814 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
18815 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
18816 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
18817 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
18818 llvm::Value *NodePtr = EmitScalarExpr(E->
getArg(0));
18819 llvm::Value *RayExtent = EmitScalarExpr(E->
getArg(1));
18820 llvm::Value *RayOrigin = EmitScalarExpr(E->
getArg(2));
18821 llvm::Value *RayDir = EmitScalarExpr(E->
getArg(3));
18822 llvm::Value *RayInverseDir = EmitScalarExpr(E->
getArg(4));
18823 llvm::Value *TextureDescr = EmitScalarExpr(E->
getArg(5));
18827 RayOrigin = Builder.CreateShuffleVector(RayOrigin, RayOrigin,
18830 Builder.CreateShuffleVector(RayDir, RayDir,
ArrayRef<int>{0, 1, 2});
18831 RayInverseDir = Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
18834 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_image_bvh_intersect_ray,
18835 {NodePtr->getType(), RayDir->getType()});
18836 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
18837 RayInverseDir, TextureDescr});
18840 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
18842 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
18843 Args.push_back(EmitScalarExpr(E->
getArg(i)));
18845 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_ds_bvh_stack_rtn);
18846 Value *Call = Builder.CreateCall(F, Args);
18847 Value *Rtn = Builder.CreateExtractValue(Call, 0);
18848 Value *A = Builder.CreateExtractValue(Call, 1);
18849 llvm::Type *RetTy = ConvertType(E->
getType());
18850 Value *I0 = Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
18852 return Builder.CreateInsertElement(I0, A, 1);
18855 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18856 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18857 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18858 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18859 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18860 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18861 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18862 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18863 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18864 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18865 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18866 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18867 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18868 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18869 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18870 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18871 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18872 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18873 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18874 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18875 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18876 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18877 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18878 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18879 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18880 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18881 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18882 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18883 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18884 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18885 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18886 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18887 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18888 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18889 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
18890 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
18891 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
18892 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
18893 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
18894 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
18895 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
18896 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
18897 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
18898 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
18899 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
18900 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
18901 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
18902 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
18903 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
18904 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
18905 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
18906 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
18907 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
18908 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
18909 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
18910 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
18911 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
18912 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
18913 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
18914 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
18927 bool AppendFalseForOpselArg =
false;
18928 unsigned BuiltinWMMAOp;
18930 switch (BuiltinID) {
18931 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18932 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18933 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18934 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18935 ArgsForMatchingMatrixTypes = {2, 0};
18936 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
18938 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18939 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18940 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18941 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18942 ArgsForMatchingMatrixTypes = {2, 0};
18943 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
18945 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18946 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18947 AppendFalseForOpselArg =
true;
18949 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18950 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18951 ArgsForMatchingMatrixTypes = {2, 0};
18952 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
18954 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18955 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18956 AppendFalseForOpselArg =
true;
18958 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18959 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18960 ArgsForMatchingMatrixTypes = {2, 0};
18961 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
18963 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18964 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18965 ArgsForMatchingMatrixTypes = {2, 0};
18966 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
18968 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18969 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18970 ArgsForMatchingMatrixTypes = {2, 0};
18971 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
18973 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18974 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18975 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18976 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18977 ArgsForMatchingMatrixTypes = {4, 1};
18978 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
18980 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18981 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18982 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18983 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18984 ArgsForMatchingMatrixTypes = {4, 1};
18985 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
18987 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18988 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18989 ArgsForMatchingMatrixTypes = {2, 0};
18990 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
18992 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18993 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18994 ArgsForMatchingMatrixTypes = {2, 0};
18995 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
18997 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18998 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18999 ArgsForMatchingMatrixTypes = {2, 0};
19000 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
19002 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
19003 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
19004 ArgsForMatchingMatrixTypes = {2, 0};
19005 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
19007 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
19008 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
19009 ArgsForMatchingMatrixTypes = {4, 1};
19010 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
19012 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
19013 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
19014 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19015 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
19017 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
19018 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
19019 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19020 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
19022 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
19023 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
19024 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19025 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
19027 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
19028 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
19029 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19030 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
19032 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
19033 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
19034 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
19035 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
19037 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
19038 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
19039 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
19040 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
19042 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
19043 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
19044 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
19045 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
19047 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
19048 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
19049 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19050 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
19052 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
19053 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
19054 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19055 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
19057 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
19058 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
19059 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19060 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
19062 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
19063 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
19064 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19065 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
19070 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
19071 Args.push_back(EmitScalarExpr(E->
getArg(i)));
19072 if (AppendFalseForOpselArg)
19073 Args.push_back(Builder.getFalse());
19076 for (
auto ArgIdx : ArgsForMatchingMatrixTypes)
19077 ArgTypes.push_back(Args[ArgIdx]->getType());
19079 Function *F = CGM.getIntrinsic(BuiltinWMMAOp, ArgTypes);
19080 return Builder.CreateCall(F, Args);
19084 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
19086 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
19088 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
19092 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
19093 return EmitAMDGPUWorkGroupSize(*
this, 0);
19094 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
19095 return EmitAMDGPUWorkGroupSize(*
this, 1);
19096 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
19097 return EmitAMDGPUWorkGroupSize(*
this, 2);
19100 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
19101 return EmitAMDGPUGridSize(*
this, 0);
19102 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
19103 return EmitAMDGPUGridSize(*
this, 1);
19104 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
19105 return EmitAMDGPUGridSize(*
this, 2);
19108 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
19109 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
19111 case AMDGPU::BI__builtin_r600_read_tidig_x:
19113 case AMDGPU::BI__builtin_r600_read_tidig_y:
19115 case AMDGPU::BI__builtin_r600_read_tidig_z:
19117 case AMDGPU::BI__builtin_amdgcn_alignbit: {
19118 llvm::Value *Src0 = EmitScalarExpr(E->
getArg(0));
19119 llvm::Value *Src1 = EmitScalarExpr(E->
getArg(1));
19120 llvm::Value *Src2 = EmitScalarExpr(E->
getArg(2));
19121 Function *F = CGM.getIntrinsic(Intrinsic::fshr, Src0->getType());
19122 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19124 case AMDGPU::BI__builtin_amdgcn_fence: {
19125 ProcessOrderScopeAMDGCN(EmitScalarExpr(E->
getArg(0)),
19126 EmitScalarExpr(E->
getArg(1)), AO, SSID);
19127 return Builder.CreateFence(AO, SSID);
19129 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
19130 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
19131 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
19132 case AMDGPU::BI__builtin_amdgcn_atomic_dec64: {
19133 llvm::AtomicRMWInst::BinOp BinOp;
19134 switch (BuiltinID) {
19135 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
19136 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
19137 BinOp = llvm::AtomicRMWInst::UIncWrap;
19139 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
19140 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
19141 BinOp = llvm::AtomicRMWInst::UDecWrap;
19148 ProcessOrderScopeAMDGCN(EmitScalarExpr(E->
getArg(2)),
19149 EmitScalarExpr(E->
getArg(3)), AO, SSID);
19155 llvm::AtomicRMWInst *RMW =
19156 Builder.CreateAtomicRMW(BinOp, Ptr, Val, AO, SSID);
19158 RMW->setVolatile(
true);
19161 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
19162 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
19163 llvm::Value *Arg = EmitScalarExpr(E->
getArg(0));
19164 llvm::Type *ResultType = ConvertType(E->
getType());
19167 CGM.getIntrinsic(Intrinsic::amdgcn_s_sendmsg_rtn, {ResultType});
19168 return Builder.CreateCall(F, {Arg});
19179 unsigned IntrinsicID,
19183 for (
unsigned I = 0; I < NumArgs; ++I)
19190 return CGF.
Builder.CreateExtractValue(Call, 0);
19195 switch (BuiltinID) {
19196 case SystemZ::BI__builtin_tbegin: {
19198 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
19199 Function *F = CGM.getIntrinsic(Intrinsic::s390_tbegin);
19200 return Builder.CreateCall(F, {TDB, Control});
19202 case SystemZ::BI__builtin_tbegin_nofloat: {
19204 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
19205 Function *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat);
19206 return Builder.CreateCall(F, {TDB, Control});
19208 case SystemZ::BI__builtin_tbeginc: {
19209 Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy);
19210 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08);
19211 Function *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc);
19212 return Builder.CreateCall(F, {TDB, Control});
19214 case SystemZ::BI__builtin_tabort: {
19216 Function *F = CGM.getIntrinsic(Intrinsic::s390_tabort);
19217 return Builder.CreateCall(F, Builder.CreateSExt(
Data, Int64Ty,
"tabort"));
19219 case SystemZ::BI__builtin_non_tx_store: {
19222 Function *F = CGM.getIntrinsic(Intrinsic::s390_ntstg);
19231 case SystemZ::BI__builtin_s390_vpopctb:
19232 case SystemZ::BI__builtin_s390_vpopcth:
19233 case SystemZ::BI__builtin_s390_vpopctf:
19234 case SystemZ::BI__builtin_s390_vpopctg: {
19235 llvm::Type *ResultType = ConvertType(E->
getType());
19237 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
19238 return Builder.CreateCall(F,
X);
19241 case SystemZ::BI__builtin_s390_vclzb:
19242 case SystemZ::BI__builtin_s390_vclzh:
19243 case SystemZ::BI__builtin_s390_vclzf:
19244 case SystemZ::BI__builtin_s390_vclzg: {
19245 llvm::Type *ResultType = ConvertType(E->
getType());
19247 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
19248 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
19249 return Builder.CreateCall(F, {
X, Undef});
19252 case SystemZ::BI__builtin_s390_vctzb:
19253 case SystemZ::BI__builtin_s390_vctzh:
19254 case SystemZ::BI__builtin_s390_vctzf:
19255 case SystemZ::BI__builtin_s390_vctzg: {
19256 llvm::Type *ResultType = ConvertType(E->
getType());
19258 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
19259 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
19260 return Builder.CreateCall(F, {
X, Undef});
19263 case SystemZ::BI__builtin_s390_verllb:
19264 case SystemZ::BI__builtin_s390_verllh:
19265 case SystemZ::BI__builtin_s390_verllf:
19266 case SystemZ::BI__builtin_s390_verllg: {
19267 llvm::Type *ResultType = ConvertType(E->
getType());
19268 llvm::Value *Src = EmitScalarExpr(E->
getArg(0));
19269 llvm::Value *Amt = EmitScalarExpr(E->
getArg(1));
19271 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
19272 Amt = Builder.CreateIntCast(Amt, ResultType->getScalarType(),
false);
19273 Amt = Builder.CreateVectorSplat(NumElts, Amt);
19274 Function *F = CGM.getIntrinsic(Intrinsic::fshl, ResultType);
19275 return Builder.CreateCall(F, { Src, Src, Amt });
19278 case SystemZ::BI__builtin_s390_verllvb:
19279 case SystemZ::BI__builtin_s390_verllvh:
19280 case SystemZ::BI__builtin_s390_verllvf:
19281 case SystemZ::BI__builtin_s390_verllvg: {
19282 llvm::Type *ResultType = ConvertType(E->
getType());
19283 llvm::Value *Src = EmitScalarExpr(E->
getArg(0));
19284 llvm::Value *Amt = EmitScalarExpr(E->
getArg(1));
19285 Function *F = CGM.getIntrinsic(Intrinsic::fshl, ResultType);
19286 return Builder.CreateCall(F, { Src, Src, Amt });
19289 case SystemZ::BI__builtin_s390_vfsqsb:
19290 case SystemZ::BI__builtin_s390_vfsqdb: {
19291 llvm::Type *ResultType = ConvertType(E->
getType());
19293 if (Builder.getIsFPConstrained()) {
19294 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt, ResultType);
19295 return Builder.CreateConstrainedFPCall(F, {
X });
19298 return Builder.CreateCall(F,
X);
19301 case SystemZ::BI__builtin_s390_vfmasb:
19302 case SystemZ::BI__builtin_s390_vfmadb: {
19303 llvm::Type *ResultType = ConvertType(E->
getType());
19307 if (Builder.getIsFPConstrained()) {
19308 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
19309 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
19312 return Builder.CreateCall(F, {
X, Y, Z});
19315 case SystemZ::BI__builtin_s390_vfmssb:
19316 case SystemZ::BI__builtin_s390_vfmsdb: {
19317 llvm::Type *ResultType = ConvertType(E->
getType());
19321 if (Builder.getIsFPConstrained()) {
19322 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
19323 return Builder.CreateConstrainedFPCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
19326 return Builder.CreateCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
19329 case SystemZ::BI__builtin_s390_vfnmasb:
19330 case SystemZ::BI__builtin_s390_vfnmadb: {
19331 llvm::Type *ResultType = ConvertType(E->
getType());
19335 if (Builder.getIsFPConstrained()) {
19336 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
19337 return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
19340 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}),
"neg");
19343 case SystemZ::BI__builtin_s390_vfnmssb:
19344 case SystemZ::BI__builtin_s390_vfnmsdb: {
19345 llvm::Type *ResultType = ConvertType(E->
getType());
19349 if (Builder.getIsFPConstrained()) {
19350 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
19351 Value *NegZ = Builder.CreateFNeg(Z,
"sub");
19352 return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
19355 Value *NegZ = Builder.CreateFNeg(Z,
"neg");
19356 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, NegZ}));
19359 case SystemZ::BI__builtin_s390_vflpsb:
19360 case SystemZ::BI__builtin_s390_vflpdb: {
19361 llvm::Type *ResultType = ConvertType(E->
getType());
19364 return Builder.CreateCall(F,
X);
19366 case SystemZ::BI__builtin_s390_vflnsb:
19367 case SystemZ::BI__builtin_s390_vflndb: {
19368 llvm::Type *ResultType = ConvertType(E->
getType());
19371 return Builder.CreateFNeg(Builder.CreateCall(F,
X),
"neg");
19373 case SystemZ::BI__builtin_s390_vfisb:
19374 case SystemZ::BI__builtin_s390_vfidb: {
19375 llvm::Type *ResultType = ConvertType(E->
getType());
19384 switch (M4.getZExtValue()) {
19387 switch (M5.getZExtValue()) {
19390 CI = Intrinsic::experimental_constrained_rint;
break;
19394 switch (M5.getZExtValue()) {
19397 CI = Intrinsic::experimental_constrained_nearbyint;
break;
19399 CI = Intrinsic::experimental_constrained_round;
break;
19401 CI = Intrinsic::experimental_constrained_trunc;
break;
19403 CI = Intrinsic::experimental_constrained_ceil;
break;
19405 CI = Intrinsic::experimental_constrained_floor;
break;
19409 if (
ID != Intrinsic::not_intrinsic) {
19410 if (Builder.getIsFPConstrained()) {
19411 Function *F = CGM.getIntrinsic(CI, ResultType);
19412 return Builder.CreateConstrainedFPCall(F,
X);
19414 Function *F = CGM.getIntrinsic(
ID, ResultType);
19415 return Builder.CreateCall(F,
X);
19418 switch (BuiltinID) {
19419 case SystemZ::BI__builtin_s390_vfisb:
ID = Intrinsic::s390_vfisb;
break;
19420 case SystemZ::BI__builtin_s390_vfidb:
ID = Intrinsic::s390_vfidb;
break;
19421 default: llvm_unreachable(
"Unknown BuiltinID");
19424 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
19425 Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5);
19426 return Builder.CreateCall(F, {
X, M4Value, M5Value});
19428 case SystemZ::BI__builtin_s390_vfmaxsb:
19429 case SystemZ::BI__builtin_s390_vfmaxdb: {
19430 llvm::Type *ResultType = ConvertType(E->
getType());
19439 switch (M4.getZExtValue()) {
19441 case 4:
ID = Intrinsic::maxnum;
19442 CI = Intrinsic::experimental_constrained_maxnum;
break;
19444 if (
ID != Intrinsic::not_intrinsic) {
19445 if (Builder.getIsFPConstrained()) {
19446 Function *F = CGM.getIntrinsic(CI, ResultType);
19447 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19449 Function *F = CGM.getIntrinsic(
ID, ResultType);
19450 return Builder.CreateCall(F, {
X, Y});
19453 switch (BuiltinID) {
19454 case SystemZ::BI__builtin_s390_vfmaxsb:
ID = Intrinsic::s390_vfmaxsb;
break;
19455 case SystemZ::BI__builtin_s390_vfmaxdb:
ID = Intrinsic::s390_vfmaxdb;
break;
19456 default: llvm_unreachable(
"Unknown BuiltinID");
19459 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
19460 return Builder.CreateCall(F, {
X, Y, M4Value});
19462 case SystemZ::BI__builtin_s390_vfminsb:
19463 case SystemZ::BI__builtin_s390_vfmindb: {
19464 llvm::Type *ResultType = ConvertType(E->
getType());
19473 switch (M4.getZExtValue()) {
19475 case 4:
ID = Intrinsic::minnum;
19476 CI = Intrinsic::experimental_constrained_minnum;
break;
19478 if (
ID != Intrinsic::not_intrinsic) {
19479 if (Builder.getIsFPConstrained()) {
19480 Function *F = CGM.getIntrinsic(CI, ResultType);
19481 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19483 Function *F = CGM.getIntrinsic(
ID, ResultType);
19484 return Builder.CreateCall(F, {
X, Y});
19487 switch (BuiltinID) {
19488 case SystemZ::BI__builtin_s390_vfminsb:
ID = Intrinsic::s390_vfminsb;
break;
19489 case SystemZ::BI__builtin_s390_vfmindb:
ID = Intrinsic::s390_vfmindb;
break;
19490 default: llvm_unreachable(
"Unknown BuiltinID");
19493 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
19494 return Builder.CreateCall(F, {
X, Y, M4Value});
19497 case SystemZ::BI__builtin_s390_vlbrh:
19498 case SystemZ::BI__builtin_s390_vlbrf:
19499 case SystemZ::BI__builtin_s390_vlbrg: {
19500 llvm::Type *ResultType = ConvertType(E->
getType());
19502 Function *F = CGM.getIntrinsic(Intrinsic::bswap, ResultType);
19503 return Builder.CreateCall(F,
X);
19508 #define INTRINSIC_WITH_CC(NAME) \
19509 case SystemZ::BI__builtin_##NAME: \
19510 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
19589 #undef INTRINSIC_WITH_CC
19598 struct NVPTXMmaLdstInfo {
19599 unsigned NumResults;
19605 #define MMA_INTR(geom_op_type, layout) \
19606 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
19607 #define MMA_LDST(n, geom_op_type) \
19608 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
19610 static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
19611 switch (BuiltinID) {
19613 case NVPTX::BI__hmma_m16n16k16_ld_a:
19614 return MMA_LDST(8, m16n16k16_load_a_f16);
19615 case NVPTX::BI__hmma_m16n16k16_ld_b:
19616 return MMA_LDST(8, m16n16k16_load_b_f16);
19617 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
19618 return MMA_LDST(4, m16n16k16_load_c_f16);
19619 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
19620 return MMA_LDST(8, m16n16k16_load_c_f32);
19621 case NVPTX::BI__hmma_m32n8k16_ld_a:
19622 return MMA_LDST(8, m32n8k16_load_a_f16);
19623 case NVPTX::BI__hmma_m32n8k16_ld_b:
19624 return MMA_LDST(8, m32n8k16_load_b_f16);
19625 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
19626 return MMA_LDST(4, m32n8k16_load_c_f16);
19627 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
19628 return MMA_LDST(8, m32n8k16_load_c_f32);
19629 case NVPTX::BI__hmma_m8n32k16_ld_a:
19630 return MMA_LDST(8, m8n32k16_load_a_f16);
19631 case NVPTX::BI__hmma_m8n32k16_ld_b:
19632 return MMA_LDST(8, m8n32k16_load_b_f16);
19633 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
19634 return MMA_LDST(4, m8n32k16_load_c_f16);
19635 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
19636 return MMA_LDST(8, m8n32k16_load_c_f32);
19639 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
19640 return MMA_LDST(2, m16n16k16_load_a_s8);
19641 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
19642 return MMA_LDST(2, m16n16k16_load_a_u8);
19643 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
19644 return MMA_LDST(2, m16n16k16_load_b_s8);
19645 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
19646 return MMA_LDST(2, m16n16k16_load_b_u8);
19647 case NVPTX::BI__imma_m16n16k16_ld_c:
19648 return MMA_LDST(8, m16n16k16_load_c_s32);
19649 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
19650 return MMA_LDST(4, m32n8k16_load_a_s8);
19651 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
19652 return MMA_LDST(4, m32n8k16_load_a_u8);
19653 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
19654 return MMA_LDST(1, m32n8k16_load_b_s8);
19655 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
19656 return MMA_LDST(1, m32n8k16_load_b_u8);
19657 case NVPTX::BI__imma_m32n8k16_ld_c:
19658 return MMA_LDST(8, m32n8k16_load_c_s32);
19659 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
19660 return MMA_LDST(1, m8n32k16_load_a_s8);
19661 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
19662 return MMA_LDST(1, m8n32k16_load_a_u8);
19663 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
19664 return MMA_LDST(4, m8n32k16_load_b_s8);
19665 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
19666 return MMA_LDST(4, m8n32k16_load_b_u8);
19667 case NVPTX::BI__imma_m8n32k16_ld_c:
19668 return MMA_LDST(8, m8n32k16_load_c_s32);
19672 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
19673 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
19674 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
19675 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
19676 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
19677 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
19678 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
19679 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
19680 case NVPTX::BI__imma_m8n8k32_ld_c:
19681 return MMA_LDST(2, m8n8k32_load_c_s32);
19682 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
19683 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
19684 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
19685 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
19686 case NVPTX::BI__bmma_m8n8k128_ld_c:
19687 return MMA_LDST(2, m8n8k128_load_c_s32);
19690 case NVPTX::BI__dmma_m8n8k4_ld_a:
19691 return MMA_LDST(1, m8n8k4_load_a_f64);
19692 case NVPTX::BI__dmma_m8n8k4_ld_b:
19693 return MMA_LDST(1, m8n8k4_load_b_f64);
19694 case NVPTX::BI__dmma_m8n8k4_ld_c:
19695 return MMA_LDST(2, m8n8k4_load_c_f64);
19698 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
19699 return MMA_LDST(4, m16n16k16_load_a_bf16);
19700 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
19701 return MMA_LDST(4, m16n16k16_load_b_bf16);
19702 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
19703 return MMA_LDST(2, m8n32k16_load_a_bf16);
19704 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
19705 return MMA_LDST(8, m8n32k16_load_b_bf16);
19706 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
19707 return MMA_LDST(8, m32n8k16_load_a_bf16);
19708 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
19709 return MMA_LDST(2, m32n8k16_load_b_bf16);
19710 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
19711 return MMA_LDST(4, m16n16k8_load_a_tf32);
19712 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
19713 return MMA_LDST(4, m16n16k8_load_b_tf32);
19714 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
19715 return MMA_LDST(8, m16n16k8_load_c_f32);
19721 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
19722 return MMA_LDST(4, m16n16k16_store_d_f16);
19723 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
19724 return MMA_LDST(8, m16n16k16_store_d_f32);
19725 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
19726 return MMA_LDST(4, m32n8k16_store_d_f16);
19727 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
19728 return MMA_LDST(8, m32n8k16_store_d_f32);
19729 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
19730 return MMA_LDST(4, m8n32k16_store_d_f16);
19731 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
19732 return MMA_LDST(8, m8n32k16_store_d_f32);
19737 case NVPTX::BI__imma_m16n16k16_st_c_i32:
19738 return MMA_LDST(8, m16n16k16_store_d_s32);
19739 case NVPTX::BI__imma_m32n8k16_st_c_i32:
19740 return MMA_LDST(8, m32n8k16_store_d_s32);
19741 case NVPTX::BI__imma_m8n32k16_st_c_i32:
19742 return MMA_LDST(8, m8n32k16_store_d_s32);
19743 case NVPTX::BI__imma_m8n8k32_st_c_i32:
19744 return MMA_LDST(2, m8n8k32_store_d_s32);
19745 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
19746 return MMA_LDST(2, m8n8k128_store_d_s32);
19749 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
19750 return MMA_LDST(2, m8n8k4_store_d_f64);
19753 case NVPTX::BI__mma_m16n16k8_st_c_f32:
19754 return MMA_LDST(8, m16n16k8_store_d_f32);
19757 llvm_unreachable(
"Unknown MMA builtin");
19764 struct NVPTXMmaInfo {
19773 std::array<unsigned, 8> Variants;
19775 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
19776 unsigned Index = Layout + 4 * Satf;
19777 if (Index >= Variants.size())
19779 return Variants[Index];
19785 static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
19787 #define MMA_VARIANTS(geom, type) \
19788 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
19789 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19790 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
19791 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
19792 #define MMA_SATF_VARIANTS(geom, type) \
19793 MMA_VARIANTS(geom, type), \
19794 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
19795 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19796 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
19797 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
19799 #define MMA_VARIANTS_I4(geom, type) \
19801 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19805 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19809 #define MMA_VARIANTS_B1_XOR(geom, type) \
19811 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
19818 #define MMA_VARIANTS_B1_AND(geom, type) \
19820 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
19828 switch (BuiltinID) {
19832 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
19834 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
19836 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
19838 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
19840 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
19842 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
19844 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
19846 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
19848 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
19850 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
19852 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
19854 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
19858 case NVPTX::BI__imma_m16n16k16_mma_s8:
19860 case NVPTX::BI__imma_m16n16k16_mma_u8:
19862 case NVPTX::BI__imma_m32n8k16_mma_s8:
19864 case NVPTX::BI__imma_m32n8k16_mma_u8:
19866 case NVPTX::BI__imma_m8n32k16_mma_s8:
19868 case NVPTX::BI__imma_m8n32k16_mma_u8:
19872 case NVPTX::BI__imma_m8n8k32_mma_s4:
19874 case NVPTX::BI__imma_m8n8k32_mma_u4:
19876 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
19878 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
19882 case NVPTX::BI__dmma_m8n8k4_mma_f64:
19886 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
19887 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
19888 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
19889 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
19890 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
19891 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
19892 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
19893 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
19895 llvm_unreachable(
"Unexpected builtin ID.");
19897 #undef MMA_VARIANTS
19898 #undef MMA_SATF_VARIANTS
19899 #undef MMA_VARIANTS_I4
19900 #undef MMA_VARIANTS_B1_AND
19901 #undef MMA_VARIANTS_B1_XOR
19910 return CGF.
Builder.CreateCall(
19912 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
19918 llvm::Type *ElemTy =
19920 return CGF.
Builder.CreateCall(
19922 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
19925 static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
19930 {CGF.EmitScalarExpr(E->getArg(0)),
19931 CGF.EmitScalarExpr(E->getArg(1)),
19932 CGF.EmitScalarExpr(E->getArg(2))})
19934 {CGF.EmitScalarExpr(E->getArg(0)),
19935 CGF.EmitScalarExpr(E->getArg(1))});
19938 static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
19941 if (!(
C.getLangOpts().NativeHalfType ||
19942 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
19944 " requires native half type support.");
19948 if (IntrinsicID == Intrinsic::nvvm_ldg_global_f ||
19949 IntrinsicID == Intrinsic::nvvm_ldu_global_f)
19950 return MakeLdgLdu(IntrinsicID, CGF, E);
19954 auto *FTy = F->getFunctionType();
19955 unsigned ICEArguments = 0;
19957 C.GetBuiltinType(BuiltinID, Error, &ICEArguments);
19959 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
19960 assert((ICEArguments & (1 << i)) == 0);
19962 auto *PTy = FTy->getParamType(i);
19963 if (PTy != ArgValue->getType())
19964 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
19965 Args.push_back(ArgValue);
19968 return CGF.
Builder.CreateCall(F, Args);
19974 auto MakeScopedLd = [&](
unsigned IntrinsicID) {
19976 llvm::Type *ElemTy =
19978 return Builder.CreateCall(
19979 CGM.getIntrinsic(IntrinsicID, {ElemTy, Ptr->getType()}), {Ptr});
19981 auto MakeScopedSt = [&](
unsigned IntrinsicID) {
19983 llvm::Type *ElemTy =
19985 return Builder.CreateCall(
19986 CGM.getIntrinsic(IntrinsicID, {Ptr->getType(), ElemTy}),
19987 {Ptr, EmitScalarExpr(E->getArg(1))});
19989 auto MakeScopedCasAtomic = [&](
unsigned IntrinsicID) {
19991 llvm::Type *ElemTy =
19993 return Builder.CreateCall(
19994 CGM.getIntrinsic(IntrinsicID, {ElemTy, Ptr->getType()}),
19995 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
19997 switch (BuiltinID) {
19999 #define LD_VOLATILE_CASES(ADDR_SPACE) \
20000 case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_i: \
20001 case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_l: \
20002 case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_ll: \
20003 return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_i_volatile); \
20004 case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_f: \
20005 case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_d: \
20006 return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_f_volatile);
20008 #define LD_CASES(ORDER, SCOPE, ADDR_SPACE) \
20009 case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_i: \
20010 case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_l: \
20011 case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_ll: \
20012 return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_i##ORDER##SCOPE); \
20013 case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_f: \
20014 case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_d: \
20015 return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_f##ORDER##SCOPE);
20017 #define LD_CASES_AS(ORDER, SCOPE) \
20018 LD_CASES(ORDER, SCOPE, _gen) \
20019 LD_CASES(ORDER, SCOPE, _global) \
20020 LD_CASES(ORDER, SCOPE, _shared)
20022 #define LD_CASES_AS_SCOPES(ORDER) \
20023 LD_CASES_AS(ORDER, ) \
20024 LD_CASES_AS(ORDER, _cta) \
20025 LD_CASES_AS(ORDER, _sys)
20033 #undef LD_VOLATILE_CASES
20036 #undef LD_CASES_AS_SCOPES
20038 #define ST_VOLATILE_CASES(ADDR_SPACE) \
20039 case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_i: \
20040 case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_l: \
20041 case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_ll: \
20042 return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_i_volatile); \
20043 case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_f: \
20044 case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_d: \
20045 return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_f_volatile);
20047 #define ST_CASES(ORDER, SCOPE, ADDR_SPACE) \
20048 case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_i: \
20049 case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_l: \
20050 case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_ll: \
20051 return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_i##ORDER##SCOPE); \
20052 case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_f: \
20053 case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_d: \
20054 return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_f##ORDER##SCOPE);
20056 #define ST_CASES_AS(ORDER, SCOPE) \
20057 ST_CASES(ORDER, SCOPE, _gen) \
20058 ST_CASES(ORDER, SCOPE, _global) \
20059 ST_CASES(ORDER, SCOPE, _shared)
20061 #define ST_CASES_AS_SCOPES(ORDER) \
20062 ST_CASES_AS(ORDER, ) \
20063 ST_CASES_AS(ORDER, _cta) \
20064 ST_CASES_AS(ORDER, _sys)
20072 #undef ST_VOLATILE_CASES
20075 #undef ST_CASES_AS_SCOPES
20077 case NVPTX::BI__nvvm_atom_add_gen_i:
20078 case NVPTX::BI__nvvm_atom_add_gen_l:
20079 case NVPTX::BI__nvvm_atom_add_gen_ll:
20082 case NVPTX::BI__nvvm_atom_sub_gen_i:
20083 case NVPTX::BI__nvvm_atom_sub_gen_l:
20084 case NVPTX::BI__nvvm_atom_sub_gen_ll:
20087 case NVPTX::BI__nvvm_atom_and_gen_i:
20088 case NVPTX::BI__nvvm_atom_and_gen_l:
20089 case NVPTX::BI__nvvm_atom_and_gen_ll:
20092 case NVPTX::BI__nvvm_atom_or_gen_i:
20093 case NVPTX::BI__nvvm_atom_or_gen_l:
20094 case NVPTX::BI__nvvm_atom_or_gen_ll:
20097 case NVPTX::BI__nvvm_atom_xor_gen_i:
20098 case NVPTX::BI__nvvm_atom_xor_gen_l:
20099 case NVPTX::BI__nvvm_atom_xor_gen_ll:
20102 case NVPTX::BI__nvvm_atom_xchg_gen_i:
20103 case NVPTX::BI__nvvm_atom_xchg_gen_l:
20104 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
20107 case NVPTX::BI__nvvm_atom_max_gen_i:
20108 case NVPTX::BI__nvvm_atom_max_gen_l:
20109 case NVPTX::BI__nvvm_atom_max_gen_ll:
20112 case NVPTX::BI__nvvm_atom_max_gen_ui:
20113 case NVPTX::BI__nvvm_atom_max_gen_ul:
20114 case NVPTX::BI__nvvm_atom_max_gen_ull:
20117 case NVPTX::BI__nvvm_atom_min_gen_i:
20118 case NVPTX::BI__nvvm_atom_min_gen_l:
20119 case NVPTX::BI__nvvm_atom_min_gen_ll:
20122 case NVPTX::BI__nvvm_atom_min_gen_ui:
20123 case NVPTX::BI__nvvm_atom_min_gen_ul:
20124 case NVPTX::BI__nvvm_atom_min_gen_ull:
20127 case NVPTX::BI__nvvm_atom_cas_gen_i:
20128 case NVPTX::BI__nvvm_atom_cas_gen_l:
20129 case NVPTX::BI__nvvm_atom_cas_gen_ll:
20134 case NVPTX::BI__nvvm_atom_add_gen_f:
20135 case NVPTX::BI__nvvm_atom_add_gen_d: {
20139 return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, DestAddr, Val,
20140 AtomicOrdering::SequentiallyConsistent);
20143 case NVPTX::BI__nvvm_atom_xchg_gen_f:
20144 case NVPTX::BI__nvvm_atom_xchg_gen_d:
20145 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f, *
this, E);
20147 case NVPTX::BI__nvvm_atom_cas_gen_f:
20148 case NVPTX::BI__nvvm_atom_cas_gen_d:
20149 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f);
20151 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
20155 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->
getType());
20156 return Builder.CreateCall(FnALI32, {Ptr, Val});
20159 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
20163 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->
getType());
20164 return Builder.CreateCall(FnALD32, {Ptr, Val});
20167 case NVPTX::BI__nvvm_ldg_c:
20168 case NVPTX::BI__nvvm_ldg_sc:
20169 case NVPTX::BI__nvvm_ldg_c2:
20170 case NVPTX::BI__nvvm_ldg_sc2:
20171 case NVPTX::BI__nvvm_ldg_c4:
20172 case NVPTX::BI__nvvm_ldg_sc4:
20173 case NVPTX::BI__nvvm_ldg_s:
20174 case NVPTX::BI__nvvm_ldg_s2:
20175 case NVPTX::BI__nvvm_ldg_s4:
20176 case NVPTX::BI__nvvm_ldg_i:
20177 case NVPTX::BI__nvvm_ldg_i2:
20178 case NVPTX::BI__nvvm_ldg_i4:
20179 case NVPTX::BI__nvvm_ldg_l:
20180 case NVPTX::BI__nvvm_ldg_l2:
20181 case NVPTX::BI__nvvm_ldg_ll:
20182 case NVPTX::BI__nvvm_ldg_ll2:
20183 case NVPTX::BI__nvvm_ldg_uc:
20184 case NVPTX::BI__nvvm_ldg_uc2:
20185 case NVPTX::BI__nvvm_ldg_uc4:
20186 case NVPTX::BI__nvvm_ldg_us:
20187 case NVPTX::BI__nvvm_ldg_us2:
20188 case NVPTX::BI__nvvm_ldg_us4:
20189 case NVPTX::BI__nvvm_ldg_ui:
20190 case NVPTX::BI__nvvm_ldg_ui2:
20191 case NVPTX::BI__nvvm_ldg_ui4:
20192 case NVPTX::BI__nvvm_ldg_ul:
20193 case NVPTX::BI__nvvm_ldg_ul2:
20194 case NVPTX::BI__nvvm_ldg_ull:
20195 case NVPTX::BI__nvvm_ldg_ull2:
20199 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_i, *
this, E);
20200 case NVPTX::BI__nvvm_ldg_f:
20201 case NVPTX::BI__nvvm_ldg_f2:
20202 case NVPTX::BI__nvvm_ldg_f4:
20203 case NVPTX::BI__nvvm_ldg_d:
20204 case NVPTX::BI__nvvm_ldg_d2:
20205 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_f, *
this, E);
20207 case NVPTX::BI__nvvm_ldu_c:
20208 case NVPTX::BI__nvvm_ldu_sc:
20209 case NVPTX::BI__nvvm_ldu_c2:
20210 case NVPTX::BI__nvvm_ldu_sc2:
20211 case NVPTX::BI__nvvm_ldu_c4:
20212 case NVPTX::BI__nvvm_ldu_sc4:
20213 case NVPTX::BI__nvvm_ldu_s:
20214 case NVPTX::BI__nvvm_ldu_s2:
20215 case NVPTX::BI__nvvm_ldu_s4:
20216 case NVPTX::BI__nvvm_ldu_i:
20217 case NVPTX::BI__nvvm_ldu_i2:
20218 case NVPTX::BI__nvvm_ldu_i4:
20219 case NVPTX::BI__nvvm_ldu_l:
20220 case NVPTX::BI__nvvm_ldu_l2:
20221 case NVPTX::BI__nvvm_ldu_ll:
20222 case NVPTX::BI__nvvm_ldu_ll2:
20223 case NVPTX::BI__nvvm_ldu_uc:
20224 case NVPTX::BI__nvvm_ldu_uc2:
20225 case NVPTX::BI__nvvm_ldu_uc4:
20226 case NVPTX::BI__nvvm_ldu_us:
20227 case NVPTX::BI__nvvm_ldu_us2:
20228 case NVPTX::BI__nvvm_ldu_us4:
20229 case NVPTX::BI__nvvm_ldu_ui:
20230 case NVPTX::BI__nvvm_ldu_ui2:
20231 case NVPTX::BI__nvvm_ldu_ui4:
20232 case NVPTX::BI__nvvm_ldu_ul:
20233 case NVPTX::BI__nvvm_ldu_ul2:
20234 case NVPTX::BI__nvvm_ldu_ull:
20235 case NVPTX::BI__nvvm_ldu_ull2:
20236 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_i, *
this, E);
20237 case NVPTX::BI__nvvm_ldu_f:
20238 case NVPTX::BI__nvvm_ldu_f2:
20239 case NVPTX::BI__nvvm_ldu_f4:
20240 case NVPTX::BI__nvvm_ldu_d:
20241 case NVPTX::BI__nvvm_ldu_d2:
20242 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_f, *
this, E);
20244 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
20245 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
20246 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
20247 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this, E);
20248 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
20249 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
20250 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
20251 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this, E);
20252 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
20253 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
20254 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this, E);
20255 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
20256 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
20257 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this, E);
20258 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
20259 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
20260 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
20261 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this, E);
20262 case NVPTX::BI__nvvm_atom_cta_xchg_gen_f:
20263 case NVPTX::BI__nvvm_atom_cta_xchg_gen_d:
20264 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_cta, *
this, E);
20265 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
20266 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
20267 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
20268 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this, E);
20269 case NVPTX::BI__nvvm_atom_sys_xchg_gen_f:
20270 case NVPTX::BI__nvvm_atom_sys_xchg_gen_d:
20271 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_sys, *
this, E);
20272 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
20273 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
20274 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
20275 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this, E);
20276 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
20277 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
20278 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
20279 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_cta, *
this, E);
20280 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
20281 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
20282 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
20283 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this, E);
20284 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
20285 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
20286 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
20287 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_sys, *
this, E);
20288 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
20289 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
20290 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
20291 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this, E);
20292 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
20293 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
20294 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
20295 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_cta, *
this, E);
20296 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
20297 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
20298 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
20299 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this, E);
20300 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
20301 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
20302 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
20303 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_sys, *
this, E);
20304 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
20305 case NVPTX::BI__nvvm_atom_cta_inc_gen_ul:
20306 case NVPTX::BI__nvvm_atom_cta_inc_gen_ull:
20307 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this, E);
20308 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
20309 case NVPTX::BI__nvvm_atom_cta_dec_gen_ul:
20310 case NVPTX::BI__nvvm_atom_cta_dec_gen_ull:
20311 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this, E);
20312 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
20313 case NVPTX::BI__nvvm_atom_sys_inc_gen_ul:
20314 case NVPTX::BI__nvvm_atom_sys_inc_gen_ull:
20315 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this, E);
20316 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
20317 case NVPTX::BI__nvvm_atom_sys_dec_gen_ul:
20318 case NVPTX::BI__nvvm_atom_sys_dec_gen_ull:
20319 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this, E);
20320 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
20321 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
20322 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
20323 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this, E);
20324 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
20325 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
20326 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
20327 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this, E);
20328 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
20329 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
20330 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
20331 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this, E);
20332 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
20333 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
20334 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
20335 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this, E);
20336 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
20337 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
20338 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
20339 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this, E);
20340 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
20341 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
20342 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
20343 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this, E);
20344 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
20345 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
20346 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
20347 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_cta);
20348 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
20349 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
20350 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
20351 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_sys);
20352 case NVPTX::BI__nvvm_atom_cta_cas_gen_f:
20353 case NVPTX::BI__nvvm_atom_cta_cas_gen_d:
20354 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_cta);
20355 case NVPTX::BI__nvvm_atom_sys_cas_gen_f:
20356 case NVPTX::BI__nvvm_atom_sys_cas_gen_d:
20357 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_sys);
20358 case NVPTX::BI__nvvm_atom_acquire_add_gen_i:
20359 case NVPTX::BI__nvvm_atom_acquire_add_gen_l:
20360 case NVPTX::BI__nvvm_atom_acquire_add_gen_ll:
20361 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_acquire, *
this, E);
20362 case NVPTX::BI__nvvm_atom_acquire_add_gen_f:
20363 case NVPTX::BI__nvvm_atom_acquire_add_gen_d:
20364 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_acquire, *
this, E);
20365 case NVPTX::BI__nvvm_atom_acquire_xchg_gen_i:
20366 case NVPTX::BI__nvvm_atom_acquire_xchg_gen_l:
20367 case NVPTX::BI__nvvm_atom_acquire_xchg_gen_ll:
20368 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_acquire, *
this, E);
20369 case NVPTX::BI__nvvm_atom_acquire_xchg_gen_f:
20370 case NVPTX::BI__nvvm_atom_acquire_xchg_gen_d:
20371 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_acquire, *
this, E);
20372 case NVPTX::BI__nvvm_atom_acquire_max_gen_i:
20373 case NVPTX::BI__nvvm_atom_acquire_max_gen_l:
20374 case NVPTX::BI__nvvm_atom_acquire_max_gen_ll:
20375 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_acquire, *
this, E);
20376 case NVPTX::BI__nvvm_atom_acquire_max_gen_ui:
20377 case NVPTX::BI__nvvm_atom_acquire_max_gen_ul:
20378 case NVPTX::BI__nvvm_atom_acquire_max_gen_ull:
20379 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_acquire, *
this, E);
20380 case NVPTX::BI__nvvm_atom_acquire_min_gen_i:
20381 case NVPTX::BI__nvvm_atom_acquire_min_gen_l:
20382 case NVPTX::BI__nvvm_atom_acquire_min_gen_ll:
20383 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_acquire, *
this, E);
20384 case NVPTX::BI__nvvm_atom_acquire_min_gen_ui:
20385 case NVPTX::BI__nvvm_atom_acquire_min_gen_ul:
20386 case NVPTX::BI__nvvm_atom_acquire_min_gen_ull:
20387 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_acquire, *
this, E);
20388 case NVPTX::BI__nvvm_atom_acquire_inc_gen_ui:
20389 case NVPTX::BI__nvvm_atom_acquire_inc_gen_ul:
20390 case NVPTX::BI__nvvm_atom_acquire_inc_gen_ull:
20391 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_acquire, *
this, E);
20392 case NVPTX::BI__nvvm_atom_acquire_dec_gen_ui:
20393 case NVPTX::BI__nvvm_atom_acquire_dec_gen_ul:
20394 case NVPTX::BI__nvvm_atom_acquire_dec_gen_ull:
20395 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_acquire, *
this, E);
20396 case NVPTX::BI__nvvm_atom_acquire_and_gen_i:
20397 case NVPTX::BI__nvvm_atom_acquire_and_gen_l:
20398 case NVPTX::BI__nvvm_atom_acquire_and_gen_ll:
20399 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_acquire, *
this, E);
20400 case NVPTX::BI__nvvm_atom_acquire_or_gen_i:
20401 case NVPTX::BI__nvvm_atom_acquire_or_gen_l:
20402 case NVPTX::BI__nvvm_atom_acquire_or_gen_ll:
20403 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_acquire, *
this, E);
20404 case NVPTX::BI__nvvm_atom_acquire_xor_gen_i:
20405 case NVPTX::BI__nvvm_atom_acquire_xor_gen_l:
20406 case NVPTX::BI__nvvm_atom_acquire_xor_gen_ll:
20407 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_acquire, *
this, E);
20408 case NVPTX::BI__nvvm_atom_acquire_cas_gen_i:
20409 case NVPTX::BI__nvvm_atom_acquire_cas_gen_l:
20410 case NVPTX::BI__nvvm_atom_acquire_cas_gen_ll:
20411 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_acquire);
20412 case NVPTX::BI__nvvm_atom_acquire_cas_gen_f:
20413 case NVPTX::BI__nvvm_atom_acquire_cas_gen_d:
20414 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_acquire);
20415 case NVPTX::BI__nvvm_atom_acquire_cta_add_gen_i:
20416 case NVPTX::BI__nvvm_atom_acquire_cta_add_gen_l:
20417 case NVPTX::BI__nvvm_atom_acquire_cta_add_gen_ll:
20418 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_acquire_cta, *
this, E);
20419 case NVPTX::BI__nvvm_atom_acquire_sys_add_gen_i:
20420 case NVPTX::BI__nvvm_atom_acquire_sys_add_gen_l:
20421 case NVPTX::BI__nvvm_atom_acquire_sys_add_gen_ll:
20422 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_acquire_sys, *
this, E);
20423 case NVPTX::BI__nvvm_atom_acquire_cta_add_gen_f:
20424 case NVPTX::BI__nvvm_atom_acquire_cta_add_gen_d:
20425 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_acquire_cta, *
this, E);
20426 case NVPTX::BI__nvvm_atom_acquire_sys_add_gen_f:
20427 case NVPTX::BI__nvvm_atom_acquire_sys_add_gen_d:
20428 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_acquire_sys, *
this, E);
20429 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_gen_i:
20430 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_gen_l:
20431 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_gen_ll:
20432 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_acquire_cta, *
this, E);
20433 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_gen_f:
20434 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_gen_d:
20435 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_acquire_cta, *
this, E);
20436 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_gen_i:
20437 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_gen_l:
20438 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_gen_ll:
20439 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_acquire_sys, *
this, E);
20440 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_gen_f:
20441 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_gen_d:
20442 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_acquire_sys, *
this, E);
20443 case NVPTX::BI__nvvm_atom_acquire_cta_max_gen_i:
20444 case NVPTX::BI__nvvm_atom_acquire_cta_max_gen_l:
20445 case NVPTX::BI__nvvm_atom_acquire_cta_max_gen_ll:
20446 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_acquire_cta, *
this, E);
20447 case NVPTX::BI__nvvm_atom_acquire_cta_max_gen_ui:
20448 case NVPTX::BI__nvvm_atom_acquire_cta_max_gen_ul:
20449 case NVPTX::BI__nvvm_atom_acquire_cta_max_gen_ull:
20450 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_acquire_cta, *
this, E);
20451 case NVPTX::BI__nvvm_atom_acquire_sys_max_gen_i:
20452 case NVPTX::BI__nvvm_atom_acquire_sys_max_gen_l:
20453 case NVPTX::BI__nvvm_atom_acquire_sys_max_gen_ll:
20454 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_acquire_sys, *
this, E);
20455 case NVPTX::BI__nvvm_atom_acquire_sys_max_gen_ui:
20456 case NVPTX::BI__nvvm_atom_acquire_sys_max_gen_ul:
20457 case NVPTX::BI__nvvm_atom_acquire_sys_max_gen_ull:
20458 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_acquire_sys, *
this, E);
20459 case NVPTX::BI__nvvm_atom_acquire_cta_min_gen_i:
20460 case NVPTX::BI__nvvm_atom_acquire_cta_min_gen_l:
20461 case NVPTX::BI__nvvm_atom_acquire_cta_min_gen_ll:
20462 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_acquire_cta, *
this, E);
20463 case NVPTX::BI__nvvm_atom_acquire_cta_min_gen_ui:
20464 case NVPTX::BI__nvvm_atom_acquire_cta_min_gen_ul:
20465 case NVPTX::BI__nvvm_atom_acquire_cta_min_gen_ull:
20466 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_acquire_cta, *
this, E);
20467 case NVPTX::BI__nvvm_atom_acquire_sys_min_gen_i:
20468 case NVPTX::BI__nvvm_atom_acquire_sys_min_gen_l:
20469 case NVPTX::BI__nvvm_atom_acquire_sys_min_gen_ll:
20470 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_acquire_sys, *
this, E);
20471 case NVPTX::BI__nvvm_atom_acquire_sys_min_gen_ui:
20472 case NVPTX::BI__nvvm_atom_acquire_sys_min_gen_ul:
20473 case NVPTX::BI__nvvm_atom_acquire_sys_min_gen_ull:
20474 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_acquire_sys, *
this, E);
20475 case NVPTX::BI__nvvm_atom_acquire_cta_inc_gen_ui:
20476 case NVPTX::BI__nvvm_atom_acquire_cta_inc_gen_ul:
20477 case NVPTX::BI__nvvm_atom_acquire_cta_inc_gen_ull:
20478 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_acquire_cta, *
this, E);
20479 case NVPTX::BI__nvvm_atom_acquire_cta_dec_gen_ui:
20480 case NVPTX::BI__nvvm_atom_acquire_cta_dec_gen_ul:
20481 case NVPTX::BI__nvvm_atom_acquire_cta_dec_gen_ull:
20482 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_acquire_cta, *
this, E);
20483 case NVPTX::BI__nvvm_atom_acquire_sys_inc_gen_ui:
20484 case NVPTX::BI__nvvm_atom_acquire_sys_inc_gen_ul:
20485 case NVPTX::BI__nvvm_atom_acquire_sys_inc_gen_ull:
20486 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_acquire_sys, *
this, E);
20487 case NVPTX::BI__nvvm_atom_acquire_sys_dec_gen_ui:
20488 case NVPTX::BI__nvvm_atom_acquire_sys_dec_gen_ul:
20489 case NVPTX::BI__nvvm_atom_acquire_sys_dec_gen_ull:
20490 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_acquire_sys, *
this, E);
20491 case NVPTX::BI__nvvm_atom_acquire_cta_and_gen_i:
20492 case NVPTX::BI__nvvm_atom_acquire_cta_and_gen_l:
20493 case NVPTX::BI__nvvm_atom_acquire_cta_and_gen_ll:
20494 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_acquire_cta, *
this, E);
20495 case NVPTX::BI__nvvm_atom_acquire_sys_and_gen_i:
20496 case NVPTX::BI__nvvm_atom_acquire_sys_and_gen_l:
20497 case NVPTX::BI__nvvm_atom_acquire_sys_and_gen_ll:
20498 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_acquire_sys, *
this, E);
20499 case NVPTX::BI__nvvm_atom_acquire_cta_or_gen_i:
20500 case NVPTX::BI__nvvm_atom_acquire_cta_or_gen_l:
20501 case NVPTX::BI__nvvm_atom_acquire_cta_or_gen_ll:
20502 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_acquire_cta, *
this, E);
20503 case NVPTX::BI__nvvm_atom_acquire_sys_or_gen_i:
20504 case NVPTX::BI__nvvm_atom_acquire_sys_or_gen_l:
20505 case NVPTX::BI__nvvm_atom_acquire_sys_or_gen_ll:
20506 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_acquire_sys, *
this, E);
20507 case NVPTX::BI__nvvm_atom_acquire_cta_xor_gen_i:
20508 case NVPTX::BI__nvvm_atom_acquire_cta_xor_gen_l:
20509 case NVPTX::BI__nvvm_atom_acquire_cta_xor_gen_ll:
20510 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_acquire_cta, *
this, E);
20511 case NVPTX::BI__nvvm_atom_acquire_sys_xor_gen_i:
20512 case NVPTX::BI__nvvm_atom_acquire_sys_xor_gen_l:
20513 case NVPTX::BI__nvvm_atom_acquire_sys_xor_gen_ll:
20514 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_acquire_sys, *
this, E);
20515 case NVPTX::BI__nvvm_atom_acquire_cta_cas_gen_i:
20516 case NVPTX::BI__nvvm_atom_acquire_cta_cas_gen_l:
20517 case NVPTX::BI__nvvm_atom_acquire_cta_cas_gen_ll:
20518 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_acquire_cta);
20519 case NVPTX::BI__nvvm_atom_acquire_sys_cas_gen_i:
20520 case NVPTX::BI__nvvm_atom_acquire_sys_cas_gen_l:
20521 case NVPTX::BI__nvvm_atom_acquire_sys_cas_gen_ll:
20522 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_acquire_sys);
20523 case NVPTX::BI__nvvm_atom_acquire_cta_cas_gen_f:
20524 case NVPTX::BI__nvvm_atom_acquire_cta_cas_gen_d:
20525 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_acquire_cta);
20526 case NVPTX::BI__nvvm_atom_acquire_sys_cas_gen_f:
20527 case NVPTX::BI__nvvm_atom_acquire_sys_cas_gen_d:
20528 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_acquire_sys);
20529 case NVPTX::BI__nvvm_atom_release_add_gen_i:
20530 case NVPTX::BI__nvvm_atom_release_add_gen_l:
20531 case NVPTX::BI__nvvm_atom_release_add_gen_ll:
20532 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_release, *
this, E);
20533 case NVPTX::BI__nvvm_atom_release_add_gen_f:
20534 case NVPTX::BI__nvvm_atom_release_add_gen_d:
20535 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_release, *
this, E);
20536 case NVPTX::BI__nvvm_atom_release_xchg_gen_i:
20537 case NVPTX::BI__nvvm_atom_release_xchg_gen_l:
20538 case NVPTX::BI__nvvm_atom_release_xchg_gen_ll:
20539 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_release, *
this, E);
20540 case NVPTX::BI__nvvm_atom_release_xchg_gen_f:
20541 case NVPTX::BI__nvvm_atom_release_xchg_gen_d:
20542 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_release, *
this, E);
20543 case NVPTX::BI__nvvm_atom_release_max_gen_i:
20544 case NVPTX::BI__nvvm_atom_release_max_gen_l:
20545 case NVPTX::BI__nvvm_atom_release_max_gen_ll:
20546 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_release, *
this, E);
20547 case NVPTX::BI__nvvm_atom_release_max_gen_ui:
20548 case NVPTX::BI__nvvm_atom_release_max_gen_ul:
20549 case NVPTX::BI__nvvm_atom_release_max_gen_ull:
20550 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_release, *
this, E);
20551 case NVPTX::BI__nvvm_atom_release_min_gen_i:
20552 case NVPTX::BI__nvvm_atom_release_min_gen_l:
20553 case NVPTX::BI__nvvm_atom_release_min_gen_ll:
20554 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_release, *
this, E);
20555 case NVPTX::BI__nvvm_atom_release_min_gen_ui:
20556 case NVPTX::BI__nvvm_atom_release_min_gen_ul:
20557 case NVPTX::BI__nvvm_atom_release_min_gen_ull:
20558 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_release, *
this, E);
20559 case NVPTX::BI__nvvm_atom_release_inc_gen_ui:
20560 case NVPTX::BI__nvvm_atom_release_inc_gen_ul:
20561 case NVPTX::BI__nvvm_atom_release_inc_gen_ull:
20562 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_release, *
this, E);
20563 case NVPTX::BI__nvvm_atom_release_dec_gen_ui:
20564 case NVPTX::BI__nvvm_atom_release_dec_gen_ul:
20565 case NVPTX::BI__nvvm_atom_release_dec_gen_ull:
20566 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_release, *
this, E);
20567 case NVPTX::BI__nvvm_atom_release_and_gen_i:
20568 case NVPTX::BI__nvvm_atom_release_and_gen_l:
20569 case NVPTX::BI__nvvm_atom_release_and_gen_ll:
20570 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_release, *
this, E);
20571 case NVPTX::BI__nvvm_atom_release_or_gen_i:
20572 case NVPTX::BI__nvvm_atom_release_or_gen_l:
20573 case NVPTX::BI__nvvm_atom_release_or_gen_ll:
20574 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_release, *
this, E);
20575 case NVPTX::BI__nvvm_atom_release_xor_gen_i:
20576 case NVPTX::BI__nvvm_atom_release_xor_gen_l:
20577 case NVPTX::BI__nvvm_atom_release_xor_gen_ll:
20578 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_release, *
this, E);
20579 case NVPTX::BI__nvvm_atom_release_cas_gen_i:
20580 case NVPTX::BI__nvvm_atom_release_cas_gen_l:
20581 case NVPTX::BI__nvvm_atom_release_cas_gen_ll:
20582 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_release);
20583 case NVPTX::BI__nvvm_atom_release_cas_gen_f:
20584 case NVPTX::BI__nvvm_atom_release_cas_gen_d:
20585 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_release);
20586 case NVPTX::BI__nvvm_atom_release_cta_add_gen_i:
20587 case NVPTX::BI__nvvm_atom_release_cta_add_gen_l:
20588 case NVPTX::BI__nvvm_atom_release_cta_add_gen_ll:
20589 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_release_cta, *
this, E);
20590 case NVPTX::BI__nvvm_atom_release_sys_add_gen_i:
20591 case NVPTX::BI__nvvm_atom_release_sys_add_gen_l:
20592 case NVPTX::BI__nvvm_atom_release_sys_add_gen_ll:
20593 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_release_sys, *
this, E);
20594 case NVPTX::BI__nvvm_atom_release_cta_add_gen_f:
20595 case NVPTX::BI__nvvm_atom_release_cta_add_gen_d:
20596 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_release_cta, *
this, E);
20597 case NVPTX::BI__nvvm_atom_release_sys_add_gen_f:
20598 case NVPTX::BI__nvvm_atom_release_sys_add_gen_d:
20599 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_release_sys, *
this, E);
20600 case NVPTX::BI__nvvm_atom_release_cta_xchg_gen_i:
20601 case NVPTX::BI__nvvm_atom_release_cta_xchg_gen_l:
20602 case NVPTX::BI__nvvm_atom_release_cta_xchg_gen_ll:
20603 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_release_cta, *
this, E);
20604 case NVPTX::BI__nvvm_atom_release_cta_xchg_gen_f:
20605 case NVPTX::BI__nvvm_atom_release_cta_xchg_gen_d:
20606 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_release_cta, *
this, E);
20607 case NVPTX::BI__nvvm_atom_release_sys_xchg_gen_i:
20608 case NVPTX::BI__nvvm_atom_release_sys_xchg_gen_l:
20609 case NVPTX::BI__nvvm_atom_release_sys_xchg_gen_ll:
20610 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_release_sys, *
this, E);
20611 case NVPTX::BI__nvvm_atom_release_sys_xchg_gen_f:
20612 case NVPTX::BI__nvvm_atom_release_sys_xchg_gen_d:
20613 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_release_sys, *
this, E);
20614 case NVPTX::BI__nvvm_atom_release_cta_max_gen_i:
20615 case NVPTX::BI__nvvm_atom_release_cta_max_gen_l:
20616 case NVPTX::BI__nvvm_atom_release_cta_max_gen_ll:
20617 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_release_cta, *
this, E);
20618 case NVPTX::BI__nvvm_atom_release_cta_max_gen_ui:
20619 case NVPTX::BI__nvvm_atom_release_cta_max_gen_ul:
20620 case NVPTX::BI__nvvm_atom_release_cta_max_gen_ull:
20621 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_release_cta, *
this, E);
20622 case NVPTX::BI__nvvm_atom_release_sys_max_gen_i:
20623 case NVPTX::BI__nvvm_atom_release_sys_max_gen_l:
20624 case NVPTX::BI__nvvm_atom_release_sys_max_gen_ll:
20625 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_release_sys, *
this, E);
20626 case NVPTX::BI__nvvm_atom_release_sys_max_gen_ui:
20627 case NVPTX::BI__nvvm_atom_release_sys_max_gen_ul:
20628 case NVPTX::BI__nvvm_atom_release_sys_max_gen_ull:
20629 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_release_sys, *
this, E);
20630 case NVPTX::BI__nvvm_atom_release_cta_min_gen_i:
20631 case NVPTX::BI__nvvm_atom_release_cta_min_gen_l:
20632 case NVPTX::BI__nvvm_atom_release_cta_min_gen_ll:
20633 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_release_cta, *
this, E);
20634 case NVPTX::BI__nvvm_atom_release_cta_min_gen_ui:
20635 case NVPTX::BI__nvvm_atom_release_cta_min_gen_ul:
20636 case NVPTX::BI__nvvm_atom_release_cta_min_gen_ull:
20637 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_release_cta, *
this, E);
20638 case NVPTX::BI__nvvm_atom_release_sys_min_gen_i:
20639 case NVPTX::BI__nvvm_atom_release_sys_min_gen_l:
20640 case NVPTX::BI__nvvm_atom_release_sys_min_gen_ll:
20641 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_release_sys, *
this, E);
20642 case NVPTX::BI__nvvm_atom_release_sys_min_gen_ui:
20643 case NVPTX::BI__nvvm_atom_release_sys_min_gen_ul:
20644 case NVPTX::BI__nvvm_atom_release_sys_min_gen_ull:
20645 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_release_sys, *
this, E);
20646 case NVPTX::BI__nvvm_atom_release_cta_inc_gen_ui:
20647 case NVPTX::BI__nvvm_atom_release_cta_inc_gen_ul:
20648 case NVPTX::BI__nvvm_atom_release_cta_inc_gen_ull:
20649 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_release_cta, *
this, E);
20650 case NVPTX::BI__nvvm_atom_release_cta_dec_gen_ui:
20651 case NVPTX::BI__nvvm_atom_release_cta_dec_gen_ul:
20652 case NVPTX::BI__nvvm_atom_release_cta_dec_gen_ull:
20653 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_release_cta, *
this, E);
20654 case NVPTX::BI__nvvm_atom_release_sys_inc_gen_ui:
20655 case NVPTX::BI__nvvm_atom_release_sys_inc_gen_ul:
20656 case NVPTX::BI__nvvm_atom_release_sys_inc_gen_ull:
20657 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_release_sys, *
this, E);
20658 case NVPTX::BI__nvvm_atom_release_sys_dec_gen_ui:
20659 case NVPTX::BI__nvvm_atom_release_sys_dec_gen_ul:
20660 case NVPTX::BI__nvvm_atom_release_sys_dec_gen_ull:
20661 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_release_sys, *
this, E);
20662 case NVPTX::BI__nvvm_atom_release_cta_and_gen_i:
20663 case NVPTX::BI__nvvm_atom_release_cta_and_gen_l:
20664 case NVPTX::BI__nvvm_atom_release_cta_and_gen_ll:
20665 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_release_cta, *
this, E);
20666 case NVPTX::BI__nvvm_atom_release_sys_and_gen_i:
20667 case NVPTX::BI__nvvm_atom_release_sys_and_gen_l:
20668 case NVPTX::BI__nvvm_atom_release_sys_and_gen_ll:
20669 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_release_sys, *
this, E);
20670 case NVPTX::BI__nvvm_atom_release_cta_or_gen_i:
20671 case NVPTX::BI__nvvm_atom_release_cta_or_gen_l:
20672 case NVPTX::BI__nvvm_atom_release_cta_or_gen_ll:
20673 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_release_cta, *
this, E);
20674 case NVPTX::BI__nvvm_atom_release_sys_or_gen_i:
20675 case NVPTX::BI__nvvm_atom_release_sys_or_gen_l:
20676 case NVPTX::BI__nvvm_atom_release_sys_or_gen_ll:
20677 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_release_sys, *
this, E);
20678 case NVPTX::BI__nvvm_atom_release_cta_xor_gen_i:
20679 case NVPTX::BI__nvvm_atom_release_cta_xor_gen_l:
20680 case NVPTX::BI__nvvm_atom_release_cta_xor_gen_ll:
20681 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_release_cta, *
this, E);
20682 case NVPTX::BI__nvvm_atom_release_sys_xor_gen_i:
20683 case NVPTX::BI__nvvm_atom_release_sys_xor_gen_l:
20684 case NVPTX::BI__nvvm_atom_release_sys_xor_gen_ll:
20685 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_release_sys, *
this, E);
20686 case NVPTX::BI__nvvm_atom_release_cta_cas_gen_i:
20687 case NVPTX::BI__nvvm_atom_release_cta_cas_gen_l:
20688 case NVPTX::BI__nvvm_atom_release_cta_cas_gen_ll:
20689 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_release_cta);
20690 case NVPTX::BI__nvvm_atom_release_sys_cas_gen_i:
20691 case NVPTX::BI__nvvm_atom_release_sys_cas_gen_l:
20692 case NVPTX::BI__nvvm_atom_release_sys_cas_gen_ll:
20693 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_release_sys);
20694 case NVPTX::BI__nvvm_atom_release_cta_cas_gen_f:
20695 case NVPTX::BI__nvvm_atom_release_cta_cas_gen_d:
20696 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_release_cta);
20697 case NVPTX::BI__nvvm_atom_release_sys_cas_gen_f:
20698 case NVPTX::BI__nvvm_atom_release_sys_cas_gen_d:
20699 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_release_sys);
20700 case NVPTX::BI__nvvm_atom_acq_rel_add_gen_i:
20701 case NVPTX::BI__nvvm_atom_acq_rel_add_gen_l:
20702 case NVPTX::BI__nvvm_atom_acq_rel_add_gen_ll:
20703 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_acq_rel, *
this, E);
20704 case NVPTX::BI__nvvm_atom_acq_rel_add_gen_f:
20705 case NVPTX::BI__nvvm_atom_acq_rel_add_gen_d:
20706 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_acq_rel, *
this, E);
20707 case NVPTX::BI__nvvm_atom_acq_rel_xchg_gen_i:
20708 case NVPTX::BI__nvvm_atom_acq_rel_xchg_gen_l:
20709 case NVPTX::BI__nvvm_atom_acq_rel_xchg_gen_ll:
20710 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_acq_rel, *
this, E);
20711 case NVPTX::BI__nvvm_atom_acq_rel_xchg_gen_f:
20712 case NVPTX::BI__nvvm_atom_acq_rel_xchg_gen_d:
20713 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_acq_rel, *
this, E);
20714 case NVPTX::BI__nvvm_atom_acq_rel_max_gen_i:
20715 case NVPTX::BI__nvvm_atom_acq_rel_max_gen_l:
20716 case NVPTX::BI__nvvm_atom_acq_rel_max_gen_ll:
20717 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_acq_rel, *
this, E);
20718 case NVPTX::BI__nvvm_atom_acq_rel_max_gen_ui:
20719 case NVPTX::BI__nvvm_atom_acq_rel_max_gen_ul:
20720 case NVPTX::BI__nvvm_atom_acq_rel_max_gen_ull:
20721 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_acq_rel, *
this, E);
20722 case NVPTX::BI__nvvm_atom_acq_rel_min_gen_i:
20723 case NVPTX::BI__nvvm_atom_acq_rel_min_gen_l:
20724 case NVPTX::BI__nvvm_atom_acq_rel_min_gen_ll:
20725 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_acq_rel, *
this, E);
20726 case NVPTX::BI__nvvm_atom_acq_rel_min_gen_ui:
20727 case NVPTX::BI__nvvm_atom_acq_rel_min_gen_ul:
20728 case NVPTX::BI__nvvm_atom_acq_rel_min_gen_ull:
20729 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_acq_rel, *
this, E);
20730 case NVPTX::BI__nvvm_atom_acq_rel_inc_gen_ui:
20731 case NVPTX::BI__nvvm_atom_acq_rel_inc_gen_ul:
20732 case NVPTX::BI__nvvm_atom_acq_rel_inc_gen_ull:
20733 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_acq_rel, *
this, E);
20734 case NVPTX::BI__nvvm_atom_acq_rel_dec_gen_ui:
20735 case NVPTX::BI__nvvm_atom_acq_rel_dec_gen_ul:
20736 case NVPTX::BI__nvvm_atom_acq_rel_dec_gen_ull:
20737 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_acq_rel, *
this, E);
20738 case NVPTX::BI__nvvm_atom_acq_rel_and_gen_i:
20739 case NVPTX::BI__nvvm_atom_acq_rel_and_gen_l:
20740 case NVPTX::BI__nvvm_atom_acq_rel_and_gen_ll:
20741 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_acq_rel, *
this, E);
20742 case NVPTX::BI__nvvm_atom_acq_rel_or_gen_i:
20743 case NVPTX::BI__nvvm_atom_acq_rel_or_gen_l:
20744 case NVPTX::BI__nvvm_atom_acq_rel_or_gen_ll:
20745 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_acq_rel, *
this, E);
20746 case NVPTX::BI__nvvm_atom_acq_rel_xor_gen_i:
20747 case NVPTX::BI__nvvm_atom_acq_rel_xor_gen_l:
20748 case NVPTX::BI__nvvm_atom_acq_rel_xor_gen_ll:
20749 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_acq_rel, *
this, E);
20750 case NVPTX::BI__nvvm_atom_acq_rel_cas_gen_i:
20751 case NVPTX::BI__nvvm_atom_acq_rel_cas_gen_l:
20752 case NVPTX::BI__nvvm_atom_acq_rel_cas_gen_ll:
20753 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_acq_rel);
20754 case NVPTX::BI__nvvm_atom_acq_rel_cas_gen_f:
20755 case NVPTX::BI__nvvm_atom_acq_rel_cas_gen_d:
20756 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_acq_rel);
20757 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_gen_i:
20758 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_gen_l:
20759 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_gen_ll:
20760 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_acq_rel_cta, *
this, E);
20761 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_gen_i:
20762 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_gen_l:
20763 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_gen_ll:
20764 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_acq_rel_sys, *
this, E);
20765 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_gen_f:
20766 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_gen_d:
20767 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_acq_rel_cta, *
this, E);
20768 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_gen_f:
20769 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_gen_d:
20770 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_acq_rel_sys, *
this, E);
20771 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_gen_i:
20772 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_gen_l:
20773 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_gen_ll:
20774 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_acq_rel_cta, *
this, E);
20775 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_gen_f:
20776 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_gen_d:
20777 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_acq_rel_cta, *
this, E);
20778 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_gen_i:
20779 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_gen_l:
20780 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_gen_ll:
20781 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_acq_rel_sys, *
this, E);
20782 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_gen_f:
20783 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_gen_d:
20784 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_f_acq_rel_sys, *
this, E);
20785 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_gen_i:
20786 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_gen_l:
20787 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_gen_ll:
20788 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_acq_rel_cta, *
this, E);
20789 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_gen_ui:
20790 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_gen_ul:
20791 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_gen_ull:
20792 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_acq_rel_cta, *
this, E);
20793 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_gen_i:
20794 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_gen_l:
20795 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_gen_ll:
20796 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_acq_rel_sys, *
this, E);
20797 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_gen_ui:
20798 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_gen_ul:
20799 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_gen_ull:
20800 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_ui_acq_rel_sys, *
this, E);
20801 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_gen_i:
20802 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_gen_l:
20803 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_gen_ll:
20804 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_acq_rel_cta, *
this, E);
20805 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_gen_ui:
20806 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_gen_ul:
20807 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_gen_ull:
20808 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_acq_rel_cta, *
this, E);
20809 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_gen_i:
20810 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_gen_l:
20811 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_gen_ll:
20812 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_acq_rel_sys, *
this, E);
20813 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_gen_ui:
20814 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_gen_ul:
20815 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_gen_ull:
20816 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_ui_acq_rel_sys, *
this, E);
20817 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_gen_ui:
20818 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_gen_ul:
20819 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_gen_ull:
20820 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_acq_rel_cta, *
this, E);
20821 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_gen_ui:
20822 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_gen_ul:
20823 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_gen_ull:
20824 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_acq_rel_cta, *
this, E);
20825 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_gen_ui:
20826 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_gen_ul:
20827 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_gen_ull:
20828 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_acq_rel_sys, *
this, E);
20829 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_gen_ui:
20830 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_gen_ul:
20831 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_gen_ull:
20832 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_acq_rel_sys, *
this, E);
20833 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_gen_i:
20834 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_gen_l:
20835 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_gen_ll:
20836 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_acq_rel_cta, *
this, E);
20837 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_gen_i:
20838 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_gen_l:
20839 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_gen_ll:
20840 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_acq_rel_sys, *
this, E);
20841 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_gen_i:
20842 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_gen_l:
20843 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_gen_ll:
20844 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_acq_rel_cta, *
this, E);
20845 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_gen_i:
20846 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_gen_l:
20847 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_gen_ll:
20848 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_acq_rel_sys, *
this, E);
20849 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_gen_i:
20850 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_gen_l:
20851 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_gen_ll:
20852 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_acq_rel_cta, *
this, E);
20853 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_gen_i:
20854 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_gen_l:
20855 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_gen_ll:
20856 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_acq_rel_sys, *
this, E);
20857 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_gen_i:
20858 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_gen_l:
20859 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_gen_ll:
20860 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_acq_rel_cta);
20861 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_gen_i:
20862 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_gen_l:
20863 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_gen_ll:
20864 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_i_acq_rel_sys);
20865 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_gen_f:
20866 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_gen_d:
20867 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_acq_rel_cta);
20868 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_gen_f:
20869 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_gen_d:
20870 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_gen_f_acq_rel_sys);
20871 case NVPTX::BI__nvvm_atom_add_global_i:
20872 case NVPTX::BI__nvvm_atom_add_global_l:
20873 case NVPTX::BI__nvvm_atom_add_global_ll:
20874 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i, *
this, E);
20875 case NVPTX::BI__nvvm_atom_add_global_f:
20876 case NVPTX::BI__nvvm_atom_add_global_d:
20877 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f, *
this, E);
20878 case NVPTX::BI__nvvm_atom_xchg_global_i:
20879 case NVPTX::BI__nvvm_atom_xchg_global_l:
20880 case NVPTX::BI__nvvm_atom_xchg_global_ll:
20881 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i, *
this, E);
20882 case NVPTX::BI__nvvm_atom_xchg_global_f:
20883 case NVPTX::BI__nvvm_atom_xchg_global_d:
20884 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f, *
this, E);
20885 case NVPTX::BI__nvvm_atom_max_global_i:
20886 case NVPTX::BI__nvvm_atom_max_global_l:
20887 case NVPTX::BI__nvvm_atom_max_global_ll:
20888 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i, *
this, E);
20889 case NVPTX::BI__nvvm_atom_max_global_ui:
20890 case NVPTX::BI__nvvm_atom_max_global_ul:
20891 case NVPTX::BI__nvvm_atom_max_global_ull:
20892 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui, *
this, E);
20893 case NVPTX::BI__nvvm_atom_min_global_i:
20894 case NVPTX::BI__nvvm_atom_min_global_l:
20895 case NVPTX::BI__nvvm_atom_min_global_ll:
20896 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i, *
this, E);
20897 case NVPTX::BI__nvvm_atom_min_global_ui:
20898 case NVPTX::BI__nvvm_atom_min_global_ul:
20899 case NVPTX::BI__nvvm_atom_min_global_ull:
20900 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui, *
this, E);
20901 case NVPTX::BI__nvvm_atom_inc_global_ui:
20902 case NVPTX::BI__nvvm_atom_inc_global_ul:
20903 case NVPTX::BI__nvvm_atom_inc_global_ull:
20904 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i, *
this, E);
20905 case NVPTX::BI__nvvm_atom_dec_global_ui:
20906 case NVPTX::BI__nvvm_atom_dec_global_ul:
20907 case NVPTX::BI__nvvm_atom_dec_global_ull:
20908 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i, *
this, E);
20909 case NVPTX::BI__nvvm_atom_and_global_i:
20910 case NVPTX::BI__nvvm_atom_and_global_l:
20911 case NVPTX::BI__nvvm_atom_and_global_ll:
20912 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i, *
this, E);
20913 case NVPTX::BI__nvvm_atom_or_global_i:
20914 case NVPTX::BI__nvvm_atom_or_global_l:
20915 case NVPTX::BI__nvvm_atom_or_global_ll:
20916 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i, *
this, E);
20917 case NVPTX::BI__nvvm_atom_xor_global_i:
20918 case NVPTX::BI__nvvm_atom_xor_global_l:
20919 case NVPTX::BI__nvvm_atom_xor_global_ll:
20920 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i, *
this, E);
20921 case NVPTX::BI__nvvm_atom_cas_global_i:
20922 case NVPTX::BI__nvvm_atom_cas_global_l:
20923 case NVPTX::BI__nvvm_atom_cas_global_ll:
20924 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i);
20925 case NVPTX::BI__nvvm_atom_cas_global_f:
20926 case NVPTX::BI__nvvm_atom_cas_global_d:
20927 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f);
20928 case NVPTX::BI__nvvm_atom_cta_add_global_i:
20929 case NVPTX::BI__nvvm_atom_cta_add_global_l:
20930 case NVPTX::BI__nvvm_atom_cta_add_global_ll:
20931 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_cta, *
this, E);
20932 case NVPTX::BI__nvvm_atom_sys_add_global_i:
20933 case NVPTX::BI__nvvm_atom_sys_add_global_l:
20934 case NVPTX::BI__nvvm_atom_sys_add_global_ll:
20935 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_sys, *
this, E);
20936 case NVPTX::BI__nvvm_atom_cta_add_global_f:
20937 case NVPTX::BI__nvvm_atom_cta_add_global_d:
20938 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_cta, *
this, E);
20939 case NVPTX::BI__nvvm_atom_sys_add_global_f:
20940 case NVPTX::BI__nvvm_atom_sys_add_global_d:
20941 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_sys, *
this, E);
20942 case NVPTX::BI__nvvm_atom_cta_xchg_global_i:
20943 case NVPTX::BI__nvvm_atom_cta_xchg_global_l:
20944 case NVPTX::BI__nvvm_atom_cta_xchg_global_ll:
20945 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_cta, *
this, E);
20946 case NVPTX::BI__nvvm_atom_cta_xchg_global_f:
20947 case NVPTX::BI__nvvm_atom_cta_xchg_global_d:
20948 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_cta, *
this, E);
20949 case NVPTX::BI__nvvm_atom_sys_xchg_global_i:
20950 case NVPTX::BI__nvvm_atom_sys_xchg_global_l:
20951 case NVPTX::BI__nvvm_atom_sys_xchg_global_ll:
20952 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_sys, *
this, E);
20953 case NVPTX::BI__nvvm_atom_sys_xchg_global_f:
20954 case NVPTX::BI__nvvm_atom_sys_xchg_global_d:
20955 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_sys, *
this, E);
20956 case NVPTX::BI__nvvm_atom_cta_max_global_i:
20957 case NVPTX::BI__nvvm_atom_cta_max_global_l:
20958 case NVPTX::BI__nvvm_atom_cta_max_global_ll:
20959 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_cta, *
this, E);
20960 case NVPTX::BI__nvvm_atom_cta_max_global_ui:
20961 case NVPTX::BI__nvvm_atom_cta_max_global_ul:
20962 case NVPTX::BI__nvvm_atom_cta_max_global_ull:
20963 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_cta, *
this, E);
20964 case NVPTX::BI__nvvm_atom_sys_max_global_i:
20965 case NVPTX::BI__nvvm_atom_sys_max_global_l:
20966 case NVPTX::BI__nvvm_atom_sys_max_global_ll:
20967 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_sys, *
this, E);
20968 case NVPTX::BI__nvvm_atom_sys_max_global_ui:
20969 case NVPTX::BI__nvvm_atom_sys_max_global_ul:
20970 case NVPTX::BI__nvvm_atom_sys_max_global_ull:
20971 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_sys, *
this, E);
20972 case NVPTX::BI__nvvm_atom_cta_min_global_i:
20973 case NVPTX::BI__nvvm_atom_cta_min_global_l:
20974 case NVPTX::BI__nvvm_atom_cta_min_global_ll:
20975 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_cta, *
this, E);
20976 case NVPTX::BI__nvvm_atom_cta_min_global_ui:
20977 case NVPTX::BI__nvvm_atom_cta_min_global_ul:
20978 case NVPTX::BI__nvvm_atom_cta_min_global_ull:
20979 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_cta, *
this, E);
20980 case NVPTX::BI__nvvm_atom_sys_min_global_i:
20981 case NVPTX::BI__nvvm_atom_sys_min_global_l:
20982 case NVPTX::BI__nvvm_atom_sys_min_global_ll:
20983 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_sys, *
this, E);
20984 case NVPTX::BI__nvvm_atom_sys_min_global_ui:
20985 case NVPTX::BI__nvvm_atom_sys_min_global_ul:
20986 case NVPTX::BI__nvvm_atom_sys_min_global_ull:
20987 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_sys, *
this, E);
20988 case NVPTX::BI__nvvm_atom_cta_inc_global_ui:
20989 case NVPTX::BI__nvvm_atom_cta_inc_global_ul:
20990 case NVPTX::BI__nvvm_atom_cta_inc_global_ull:
20991 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_cta, *
this, E);
20992 case NVPTX::BI__nvvm_atom_cta_dec_global_ui:
20993 case NVPTX::BI__nvvm_atom_cta_dec_global_ul:
20994 case NVPTX::BI__nvvm_atom_cta_dec_global_ull:
20995 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_cta, *
this, E);
20996 case NVPTX::BI__nvvm_atom_sys_inc_global_ui:
20997 case NVPTX::BI__nvvm_atom_sys_inc_global_ul:
20998 case NVPTX::BI__nvvm_atom_sys_inc_global_ull:
20999 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_sys, *
this, E);
21000 case NVPTX::BI__nvvm_atom_sys_dec_global_ui:
21001 case NVPTX::BI__nvvm_atom_sys_dec_global_ul:
21002 case NVPTX::BI__nvvm_atom_sys_dec_global_ull:
21003 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_sys, *
this, E);
21004 case NVPTX::BI__nvvm_atom_cta_and_global_i:
21005 case NVPTX::BI__nvvm_atom_cta_and_global_l:
21006 case NVPTX::BI__nvvm_atom_cta_and_global_ll:
21007 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_cta, *
this, E);
21008 case NVPTX::BI__nvvm_atom_sys_and_global_i:
21009 case NVPTX::BI__nvvm_atom_sys_and_global_l:
21010 case NVPTX::BI__nvvm_atom_sys_and_global_ll:
21011 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_sys, *
this, E);
21012 case NVPTX::BI__nvvm_atom_cta_or_global_i:
21013 case NVPTX::BI__nvvm_atom_cta_or_global_l:
21014 case NVPTX::BI__nvvm_atom_cta_or_global_ll:
21015 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_cta, *
this, E);
21016 case NVPTX::BI__nvvm_atom_sys_or_global_i:
21017 case NVPTX::BI__nvvm_atom_sys_or_global_l:
21018 case NVPTX::BI__nvvm_atom_sys_or_global_ll:
21019 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_sys, *
this, E);
21020 case NVPTX::BI__nvvm_atom_cta_xor_global_i:
21021 case NVPTX::BI__nvvm_atom_cta_xor_global_l:
21022 case NVPTX::BI__nvvm_atom_cta_xor_global_ll:
21023 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_cta, *
this, E);
21024 case NVPTX::BI__nvvm_atom_sys_xor_global_i:
21025 case NVPTX::BI__nvvm_atom_sys_xor_global_l:
21026 case NVPTX::BI__nvvm_atom_sys_xor_global_ll:
21027 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_sys, *
this, E);
21028 case NVPTX::BI__nvvm_atom_cta_cas_global_i:
21029 case NVPTX::BI__nvvm_atom_cta_cas_global_l:
21030 case NVPTX::BI__nvvm_atom_cta_cas_global_ll:
21031 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_cta);
21032 case NVPTX::BI__nvvm_atom_sys_cas_global_i:
21033 case NVPTX::BI__nvvm_atom_sys_cas_global_l:
21034 case NVPTX::BI__nvvm_atom_sys_cas_global_ll:
21035 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_sys);
21036 case NVPTX::BI__nvvm_atom_cta_cas_global_f:
21037 case NVPTX::BI__nvvm_atom_cta_cas_global_d:
21038 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_cta);
21039 case NVPTX::BI__nvvm_atom_sys_cas_global_f:
21040 case NVPTX::BI__nvvm_atom_sys_cas_global_d:
21041 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_sys);
21042 case NVPTX::BI__nvvm_atom_acquire_add_global_i:
21043 case NVPTX::BI__nvvm_atom_acquire_add_global_l:
21044 case NVPTX::BI__nvvm_atom_acquire_add_global_ll:
21045 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_acquire, *
this, E);
21046 case NVPTX::BI__nvvm_atom_acquire_add_global_f:
21047 case NVPTX::BI__nvvm_atom_acquire_add_global_d:
21048 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_acquire, *
this, E);
21049 case NVPTX::BI__nvvm_atom_acquire_xchg_global_i:
21050 case NVPTX::BI__nvvm_atom_acquire_xchg_global_l:
21051 case NVPTX::BI__nvvm_atom_acquire_xchg_global_ll:
21052 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_acquire, *
this, E);
21053 case NVPTX::BI__nvvm_atom_acquire_xchg_global_f:
21054 case NVPTX::BI__nvvm_atom_acquire_xchg_global_d:
21055 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_acquire, *
this, E);
21056 case NVPTX::BI__nvvm_atom_acquire_max_global_i:
21057 case NVPTX::BI__nvvm_atom_acquire_max_global_l:
21058 case NVPTX::BI__nvvm_atom_acquire_max_global_ll:
21059 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_acquire, *
this, E);
21060 case NVPTX::BI__nvvm_atom_acquire_max_global_ui:
21061 case NVPTX::BI__nvvm_atom_acquire_max_global_ul:
21062 case NVPTX::BI__nvvm_atom_acquire_max_global_ull:
21063 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_acquire, *
this, E);
21064 case NVPTX::BI__nvvm_atom_acquire_min_global_i:
21065 case NVPTX::BI__nvvm_atom_acquire_min_global_l:
21066 case NVPTX::BI__nvvm_atom_acquire_min_global_ll:
21067 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_acquire, *
this, E);
21068 case NVPTX::BI__nvvm_atom_acquire_min_global_ui:
21069 case NVPTX::BI__nvvm_atom_acquire_min_global_ul:
21070 case NVPTX::BI__nvvm_atom_acquire_min_global_ull:
21071 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_acquire, *
this, E);
21072 case NVPTX::BI__nvvm_atom_acquire_inc_global_ui:
21073 case NVPTX::BI__nvvm_atom_acquire_inc_global_ul:
21074 case NVPTX::BI__nvvm_atom_acquire_inc_global_ull:
21075 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_acquire, *
this, E);
21076 case NVPTX::BI__nvvm_atom_acquire_dec_global_ui:
21077 case NVPTX::BI__nvvm_atom_acquire_dec_global_ul:
21078 case NVPTX::BI__nvvm_atom_acquire_dec_global_ull:
21079 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_acquire, *
this, E);
21080 case NVPTX::BI__nvvm_atom_acquire_and_global_i:
21081 case NVPTX::BI__nvvm_atom_acquire_and_global_l:
21082 case NVPTX::BI__nvvm_atom_acquire_and_global_ll:
21083 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_acquire, *
this, E);
21084 case NVPTX::BI__nvvm_atom_acquire_or_global_i:
21085 case NVPTX::BI__nvvm_atom_acquire_or_global_l:
21086 case NVPTX::BI__nvvm_atom_acquire_or_global_ll:
21087 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_acquire, *
this, E);
21088 case NVPTX::BI__nvvm_atom_acquire_xor_global_i:
21089 case NVPTX::BI__nvvm_atom_acquire_xor_global_l:
21090 case NVPTX::BI__nvvm_atom_acquire_xor_global_ll:
21091 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_acquire, *
this, E);
21092 case NVPTX::BI__nvvm_atom_acquire_cas_global_i:
21093 case NVPTX::BI__nvvm_atom_acquire_cas_global_l:
21094 case NVPTX::BI__nvvm_atom_acquire_cas_global_ll:
21095 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_acquire);
21096 case NVPTX::BI__nvvm_atom_acquire_cas_global_f:
21097 case NVPTX::BI__nvvm_atom_acquire_cas_global_d:
21098 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_acquire);
21099 case NVPTX::BI__nvvm_atom_acquire_cta_add_global_i:
21100 case NVPTX::BI__nvvm_atom_acquire_cta_add_global_l:
21101 case NVPTX::BI__nvvm_atom_acquire_cta_add_global_ll:
21102 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_acquire_cta, *
this, E);
21103 case NVPTX::BI__nvvm_atom_acquire_sys_add_global_i:
21104 case NVPTX::BI__nvvm_atom_acquire_sys_add_global_l:
21105 case NVPTX::BI__nvvm_atom_acquire_sys_add_global_ll:
21106 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_acquire_sys, *
this, E);
21107 case NVPTX::BI__nvvm_atom_acquire_cta_add_global_f:
21108 case NVPTX::BI__nvvm_atom_acquire_cta_add_global_d:
21109 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_acquire_cta, *
this, E);
21110 case NVPTX::BI__nvvm_atom_acquire_sys_add_global_f:
21111 case NVPTX::BI__nvvm_atom_acquire_sys_add_global_d:
21112 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_acquire_sys, *
this, E);
21113 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_global_i:
21114 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_global_l:
21115 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_global_ll:
21116 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_acquire_cta, *
this, E);
21117 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_global_f:
21118 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_global_d:
21119 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_acquire_cta, *
this, E);
21120 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_global_i:
21121 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_global_l:
21122 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_global_ll:
21123 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_acquire_sys, *
this, E);
21124 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_global_f:
21125 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_global_d:
21126 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_acquire_sys, *
this, E);
21127 case NVPTX::BI__nvvm_atom_acquire_cta_max_global_i:
21128 case NVPTX::BI__nvvm_atom_acquire_cta_max_global_l:
21129 case NVPTX::BI__nvvm_atom_acquire_cta_max_global_ll:
21130 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_acquire_cta, *
this, E);
21131 case NVPTX::BI__nvvm_atom_acquire_cta_max_global_ui:
21132 case NVPTX::BI__nvvm_atom_acquire_cta_max_global_ul:
21133 case NVPTX::BI__nvvm_atom_acquire_cta_max_global_ull:
21134 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_acquire_cta, *
this, E);
21135 case NVPTX::BI__nvvm_atom_acquire_sys_max_global_i:
21136 case NVPTX::BI__nvvm_atom_acquire_sys_max_global_l:
21137 case NVPTX::BI__nvvm_atom_acquire_sys_max_global_ll:
21138 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_acquire_sys, *
this, E);
21139 case NVPTX::BI__nvvm_atom_acquire_sys_max_global_ui:
21140 case NVPTX::BI__nvvm_atom_acquire_sys_max_global_ul:
21141 case NVPTX::BI__nvvm_atom_acquire_sys_max_global_ull:
21142 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_acquire_sys, *
this, E);
21143 case NVPTX::BI__nvvm_atom_acquire_cta_min_global_i:
21144 case NVPTX::BI__nvvm_atom_acquire_cta_min_global_l:
21145 case NVPTX::BI__nvvm_atom_acquire_cta_min_global_ll:
21146 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_acquire_cta, *
this, E);
21147 case NVPTX::BI__nvvm_atom_acquire_cta_min_global_ui:
21148 case NVPTX::BI__nvvm_atom_acquire_cta_min_global_ul:
21149 case NVPTX::BI__nvvm_atom_acquire_cta_min_global_ull:
21150 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_acquire_cta, *
this, E);
21151 case NVPTX::BI__nvvm_atom_acquire_sys_min_global_i:
21152 case NVPTX::BI__nvvm_atom_acquire_sys_min_global_l:
21153 case NVPTX::BI__nvvm_atom_acquire_sys_min_global_ll:
21154 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_acquire_sys, *
this, E);
21155 case NVPTX::BI__nvvm_atom_acquire_sys_min_global_ui:
21156 case NVPTX::BI__nvvm_atom_acquire_sys_min_global_ul:
21157 case NVPTX::BI__nvvm_atom_acquire_sys_min_global_ull:
21158 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_acquire_sys, *
this, E);
21159 case NVPTX::BI__nvvm_atom_acquire_cta_inc_global_ui:
21160 case NVPTX::BI__nvvm_atom_acquire_cta_inc_global_ul:
21161 case NVPTX::BI__nvvm_atom_acquire_cta_inc_global_ull:
21162 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_acquire_cta, *
this, E);
21163 case NVPTX::BI__nvvm_atom_acquire_cta_dec_global_ui:
21164 case NVPTX::BI__nvvm_atom_acquire_cta_dec_global_ul:
21165 case NVPTX::BI__nvvm_atom_acquire_cta_dec_global_ull:
21166 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_acquire_cta, *
this, E);
21167 case NVPTX::BI__nvvm_atom_acquire_sys_inc_global_ui:
21168 case NVPTX::BI__nvvm_atom_acquire_sys_inc_global_ul:
21169 case NVPTX::BI__nvvm_atom_acquire_sys_inc_global_ull:
21170 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_acquire_sys, *
this, E);
21171 case NVPTX::BI__nvvm_atom_acquire_sys_dec_global_ui:
21172 case NVPTX::BI__nvvm_atom_acquire_sys_dec_global_ul:
21173 case NVPTX::BI__nvvm_atom_acquire_sys_dec_global_ull:
21174 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_acquire_sys, *
this, E);
21175 case NVPTX::BI__nvvm_atom_acquire_cta_and_global_i:
21176 case NVPTX::BI__nvvm_atom_acquire_cta_and_global_l:
21177 case NVPTX::BI__nvvm_atom_acquire_cta_and_global_ll:
21178 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_acquire_cta, *
this, E);
21179 case NVPTX::BI__nvvm_atom_acquire_sys_and_global_i:
21180 case NVPTX::BI__nvvm_atom_acquire_sys_and_global_l:
21181 case NVPTX::BI__nvvm_atom_acquire_sys_and_global_ll:
21182 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_acquire_sys, *
this, E);
21183 case NVPTX::BI__nvvm_atom_acquire_cta_or_global_i:
21184 case NVPTX::BI__nvvm_atom_acquire_cta_or_global_l:
21185 case NVPTX::BI__nvvm_atom_acquire_cta_or_global_ll:
21186 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_acquire_cta, *
this, E);
21187 case NVPTX::BI__nvvm_atom_acquire_sys_or_global_i:
21188 case NVPTX::BI__nvvm_atom_acquire_sys_or_global_l:
21189 case NVPTX::BI__nvvm_atom_acquire_sys_or_global_ll:
21190 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_acquire_sys, *
this, E);
21191 case NVPTX::BI__nvvm_atom_acquire_cta_xor_global_i:
21192 case NVPTX::BI__nvvm_atom_acquire_cta_xor_global_l:
21193 case NVPTX::BI__nvvm_atom_acquire_cta_xor_global_ll:
21194 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_acquire_cta, *
this, E);
21195 case NVPTX::BI__nvvm_atom_acquire_sys_xor_global_i:
21196 case NVPTX::BI__nvvm_atom_acquire_sys_xor_global_l:
21197 case NVPTX::BI__nvvm_atom_acquire_sys_xor_global_ll:
21198 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_acquire_sys, *
this, E);
21199 case NVPTX::BI__nvvm_atom_acquire_cta_cas_global_i:
21200 case NVPTX::BI__nvvm_atom_acquire_cta_cas_global_l:
21201 case NVPTX::BI__nvvm_atom_acquire_cta_cas_global_ll:
21202 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_acquire_cta);
21203 case NVPTX::BI__nvvm_atom_acquire_sys_cas_global_i:
21204 case NVPTX::BI__nvvm_atom_acquire_sys_cas_global_l:
21205 case NVPTX::BI__nvvm_atom_acquire_sys_cas_global_ll:
21206 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_acquire_sys);
21207 case NVPTX::BI__nvvm_atom_acquire_cta_cas_global_f:
21208 case NVPTX::BI__nvvm_atom_acquire_cta_cas_global_d:
21209 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_acquire_cta);
21210 case NVPTX::BI__nvvm_atom_acquire_sys_cas_global_f:
21211 case NVPTX::BI__nvvm_atom_acquire_sys_cas_global_d:
21212 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_acquire_sys);
21213 case NVPTX::BI__nvvm_atom_release_add_global_i:
21214 case NVPTX::BI__nvvm_atom_release_add_global_l:
21215 case NVPTX::BI__nvvm_atom_release_add_global_ll:
21216 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_release, *
this, E);
21217 case NVPTX::BI__nvvm_atom_release_add_global_f:
21218 case NVPTX::BI__nvvm_atom_release_add_global_d:
21219 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_release, *
this, E);
21220 case NVPTX::BI__nvvm_atom_release_xchg_global_i:
21221 case NVPTX::BI__nvvm_atom_release_xchg_global_l:
21222 case NVPTX::BI__nvvm_atom_release_xchg_global_ll:
21223 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_release, *
this, E);
21224 case NVPTX::BI__nvvm_atom_release_xchg_global_f:
21225 case NVPTX::BI__nvvm_atom_release_xchg_global_d:
21226 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_release, *
this, E);
21227 case NVPTX::BI__nvvm_atom_release_max_global_i:
21228 case NVPTX::BI__nvvm_atom_release_max_global_l:
21229 case NVPTX::BI__nvvm_atom_release_max_global_ll:
21230 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_release, *
this, E);
21231 case NVPTX::BI__nvvm_atom_release_max_global_ui:
21232 case NVPTX::BI__nvvm_atom_release_max_global_ul:
21233 case NVPTX::BI__nvvm_atom_release_max_global_ull:
21234 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_release, *
this, E);
21235 case NVPTX::BI__nvvm_atom_release_min_global_i:
21236 case NVPTX::BI__nvvm_atom_release_min_global_l:
21237 case NVPTX::BI__nvvm_atom_release_min_global_ll:
21238 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_release, *
this, E);
21239 case NVPTX::BI__nvvm_atom_release_min_global_ui:
21240 case NVPTX::BI__nvvm_atom_release_min_global_ul:
21241 case NVPTX::BI__nvvm_atom_release_min_global_ull:
21242 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_release, *
this, E);
21243 case NVPTX::BI__nvvm_atom_release_inc_global_ui:
21244 case NVPTX::BI__nvvm_atom_release_inc_global_ul:
21245 case NVPTX::BI__nvvm_atom_release_inc_global_ull:
21246 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_release, *
this, E);
21247 case NVPTX::BI__nvvm_atom_release_dec_global_ui:
21248 case NVPTX::BI__nvvm_atom_release_dec_global_ul:
21249 case NVPTX::BI__nvvm_atom_release_dec_global_ull:
21250 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_release, *
this, E);
21251 case NVPTX::BI__nvvm_atom_release_and_global_i:
21252 case NVPTX::BI__nvvm_atom_release_and_global_l:
21253 case NVPTX::BI__nvvm_atom_release_and_global_ll:
21254 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_release, *
this, E);
21255 case NVPTX::BI__nvvm_atom_release_or_global_i:
21256 case NVPTX::BI__nvvm_atom_release_or_global_l:
21257 case NVPTX::BI__nvvm_atom_release_or_global_ll:
21258 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_release, *
this, E);
21259 case NVPTX::BI__nvvm_atom_release_xor_global_i:
21260 case NVPTX::BI__nvvm_atom_release_xor_global_l:
21261 case NVPTX::BI__nvvm_atom_release_xor_global_ll:
21262 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_release, *
this, E);
21263 case NVPTX::BI__nvvm_atom_release_cas_global_i:
21264 case NVPTX::BI__nvvm_atom_release_cas_global_l:
21265 case NVPTX::BI__nvvm_atom_release_cas_global_ll:
21266 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_release);
21267 case NVPTX::BI__nvvm_atom_release_cas_global_f:
21268 case NVPTX::BI__nvvm_atom_release_cas_global_d:
21269 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_release);
21270 case NVPTX::BI__nvvm_atom_release_cta_add_global_i:
21271 case NVPTX::BI__nvvm_atom_release_cta_add_global_l:
21272 case NVPTX::BI__nvvm_atom_release_cta_add_global_ll:
21273 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_release_cta, *
this, E);
21274 case NVPTX::BI__nvvm_atom_release_sys_add_global_i:
21275 case NVPTX::BI__nvvm_atom_release_sys_add_global_l:
21276 case NVPTX::BI__nvvm_atom_release_sys_add_global_ll:
21277 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_release_sys, *
this, E);
21278 case NVPTX::BI__nvvm_atom_release_cta_add_global_f:
21279 case NVPTX::BI__nvvm_atom_release_cta_add_global_d:
21280 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_release_cta, *
this, E);
21281 case NVPTX::BI__nvvm_atom_release_sys_add_global_f:
21282 case NVPTX::BI__nvvm_atom_release_sys_add_global_d:
21283 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_release_sys, *
this, E);
21284 case NVPTX::BI__nvvm_atom_release_cta_xchg_global_i:
21285 case NVPTX::BI__nvvm_atom_release_cta_xchg_global_l:
21286 case NVPTX::BI__nvvm_atom_release_cta_xchg_global_ll:
21287 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_release_cta, *
this, E);
21288 case NVPTX::BI__nvvm_atom_release_cta_xchg_global_f:
21289 case NVPTX::BI__nvvm_atom_release_cta_xchg_global_d:
21290 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_release_cta, *
this, E);
21291 case NVPTX::BI__nvvm_atom_release_sys_xchg_global_i:
21292 case NVPTX::BI__nvvm_atom_release_sys_xchg_global_l:
21293 case NVPTX::BI__nvvm_atom_release_sys_xchg_global_ll:
21294 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_release_sys, *
this, E);
21295 case NVPTX::BI__nvvm_atom_release_sys_xchg_global_f:
21296 case NVPTX::BI__nvvm_atom_release_sys_xchg_global_d:
21297 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_release_sys, *
this, E);
21298 case NVPTX::BI__nvvm_atom_release_cta_max_global_i:
21299 case NVPTX::BI__nvvm_atom_release_cta_max_global_l:
21300 case NVPTX::BI__nvvm_atom_release_cta_max_global_ll:
21301 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_release_cta, *
this, E);
21302 case NVPTX::BI__nvvm_atom_release_cta_max_global_ui:
21303 case NVPTX::BI__nvvm_atom_release_cta_max_global_ul:
21304 case NVPTX::BI__nvvm_atom_release_cta_max_global_ull:
21305 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_release_cta, *
this, E);
21306 case NVPTX::BI__nvvm_atom_release_sys_max_global_i:
21307 case NVPTX::BI__nvvm_atom_release_sys_max_global_l:
21308 case NVPTX::BI__nvvm_atom_release_sys_max_global_ll:
21309 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_release_sys, *
this, E);
21310 case NVPTX::BI__nvvm_atom_release_sys_max_global_ui:
21311 case NVPTX::BI__nvvm_atom_release_sys_max_global_ul:
21312 case NVPTX::BI__nvvm_atom_release_sys_max_global_ull:
21313 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_release_sys, *
this, E);
21314 case NVPTX::BI__nvvm_atom_release_cta_min_global_i:
21315 case NVPTX::BI__nvvm_atom_release_cta_min_global_l:
21316 case NVPTX::BI__nvvm_atom_release_cta_min_global_ll:
21317 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_release_cta, *
this, E);
21318 case NVPTX::BI__nvvm_atom_release_cta_min_global_ui:
21319 case NVPTX::BI__nvvm_atom_release_cta_min_global_ul:
21320 case NVPTX::BI__nvvm_atom_release_cta_min_global_ull:
21321 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_release_cta, *
this, E);
21322 case NVPTX::BI__nvvm_atom_release_sys_min_global_i:
21323 case NVPTX::BI__nvvm_atom_release_sys_min_global_l:
21324 case NVPTX::BI__nvvm_atom_release_sys_min_global_ll:
21325 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_release_sys, *
this, E);
21326 case NVPTX::BI__nvvm_atom_release_sys_min_global_ui:
21327 case NVPTX::BI__nvvm_atom_release_sys_min_global_ul:
21328 case NVPTX::BI__nvvm_atom_release_sys_min_global_ull:
21329 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_release_sys, *
this, E);
21330 case NVPTX::BI__nvvm_atom_release_cta_inc_global_ui:
21331 case NVPTX::BI__nvvm_atom_release_cta_inc_global_ul:
21332 case NVPTX::BI__nvvm_atom_release_cta_inc_global_ull:
21333 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_release_cta, *
this, E);
21334 case NVPTX::BI__nvvm_atom_release_cta_dec_global_ui:
21335 case NVPTX::BI__nvvm_atom_release_cta_dec_global_ul:
21336 case NVPTX::BI__nvvm_atom_release_cta_dec_global_ull:
21337 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_release_cta, *
this, E);
21338 case NVPTX::BI__nvvm_atom_release_sys_inc_global_ui:
21339 case NVPTX::BI__nvvm_atom_release_sys_inc_global_ul:
21340 case NVPTX::BI__nvvm_atom_release_sys_inc_global_ull:
21341 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_release_sys, *
this, E);
21342 case NVPTX::BI__nvvm_atom_release_sys_dec_global_ui:
21343 case NVPTX::BI__nvvm_atom_release_sys_dec_global_ul:
21344 case NVPTX::BI__nvvm_atom_release_sys_dec_global_ull:
21345 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_release_sys, *
this, E);
21346 case NVPTX::BI__nvvm_atom_release_cta_and_global_i:
21347 case NVPTX::BI__nvvm_atom_release_cta_and_global_l:
21348 case NVPTX::BI__nvvm_atom_release_cta_and_global_ll:
21349 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_release_cta, *
this, E);
21350 case NVPTX::BI__nvvm_atom_release_sys_and_global_i:
21351 case NVPTX::BI__nvvm_atom_release_sys_and_global_l:
21352 case NVPTX::BI__nvvm_atom_release_sys_and_global_ll:
21353 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_release_sys, *
this, E);
21354 case NVPTX::BI__nvvm_atom_release_cta_or_global_i:
21355 case NVPTX::BI__nvvm_atom_release_cta_or_global_l:
21356 case NVPTX::BI__nvvm_atom_release_cta_or_global_ll:
21357 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_release_cta, *
this, E);
21358 case NVPTX::BI__nvvm_atom_release_sys_or_global_i:
21359 case NVPTX::BI__nvvm_atom_release_sys_or_global_l:
21360 case NVPTX::BI__nvvm_atom_release_sys_or_global_ll:
21361 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_release_sys, *
this, E);
21362 case NVPTX::BI__nvvm_atom_release_cta_xor_global_i:
21363 case NVPTX::BI__nvvm_atom_release_cta_xor_global_l:
21364 case NVPTX::BI__nvvm_atom_release_cta_xor_global_ll:
21365 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_release_cta, *
this, E);
21366 case NVPTX::BI__nvvm_atom_release_sys_xor_global_i:
21367 case NVPTX::BI__nvvm_atom_release_sys_xor_global_l:
21368 case NVPTX::BI__nvvm_atom_release_sys_xor_global_ll:
21369 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_release_sys, *
this, E);
21370 case NVPTX::BI__nvvm_atom_release_cta_cas_global_i:
21371 case NVPTX::BI__nvvm_atom_release_cta_cas_global_l:
21372 case NVPTX::BI__nvvm_atom_release_cta_cas_global_ll:
21373 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_release_cta);
21374 case NVPTX::BI__nvvm_atom_release_sys_cas_global_i:
21375 case NVPTX::BI__nvvm_atom_release_sys_cas_global_l:
21376 case NVPTX::BI__nvvm_atom_release_sys_cas_global_ll:
21377 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_release_sys);
21378 case NVPTX::BI__nvvm_atom_release_cta_cas_global_f:
21379 case NVPTX::BI__nvvm_atom_release_cta_cas_global_d:
21380 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_release_cta);
21381 case NVPTX::BI__nvvm_atom_release_sys_cas_global_f:
21382 case NVPTX::BI__nvvm_atom_release_sys_cas_global_d:
21383 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_release_sys);
21384 case NVPTX::BI__nvvm_atom_acq_rel_add_global_i:
21385 case NVPTX::BI__nvvm_atom_acq_rel_add_global_l:
21386 case NVPTX::BI__nvvm_atom_acq_rel_add_global_ll:
21387 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_acq_rel, *
this, E);
21388 case NVPTX::BI__nvvm_atom_acq_rel_add_global_f:
21389 case NVPTX::BI__nvvm_atom_acq_rel_add_global_d:
21390 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_acq_rel, *
this, E);
21391 case NVPTX::BI__nvvm_atom_acq_rel_xchg_global_i:
21392 case NVPTX::BI__nvvm_atom_acq_rel_xchg_global_l:
21393 case NVPTX::BI__nvvm_atom_acq_rel_xchg_global_ll:
21394 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_acq_rel, *
this, E);
21395 case NVPTX::BI__nvvm_atom_acq_rel_xchg_global_f:
21396 case NVPTX::BI__nvvm_atom_acq_rel_xchg_global_d:
21397 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_acq_rel, *
this, E);
21398 case NVPTX::BI__nvvm_atom_acq_rel_max_global_i:
21399 case NVPTX::BI__nvvm_atom_acq_rel_max_global_l:
21400 case NVPTX::BI__nvvm_atom_acq_rel_max_global_ll:
21401 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_acq_rel, *
this, E);
21402 case NVPTX::BI__nvvm_atom_acq_rel_max_global_ui:
21403 case NVPTX::BI__nvvm_atom_acq_rel_max_global_ul:
21404 case NVPTX::BI__nvvm_atom_acq_rel_max_global_ull:
21405 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_acq_rel, *
this, E);
21406 case NVPTX::BI__nvvm_atom_acq_rel_min_global_i:
21407 case NVPTX::BI__nvvm_atom_acq_rel_min_global_l:
21408 case NVPTX::BI__nvvm_atom_acq_rel_min_global_ll:
21409 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_acq_rel, *
this, E);
21410 case NVPTX::BI__nvvm_atom_acq_rel_min_global_ui:
21411 case NVPTX::BI__nvvm_atom_acq_rel_min_global_ul:
21412 case NVPTX::BI__nvvm_atom_acq_rel_min_global_ull:
21413 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_acq_rel, *
this, E);
21414 case NVPTX::BI__nvvm_atom_acq_rel_inc_global_ui:
21415 case NVPTX::BI__nvvm_atom_acq_rel_inc_global_ul:
21416 case NVPTX::BI__nvvm_atom_acq_rel_inc_global_ull:
21417 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_acq_rel, *
this, E);
21418 case NVPTX::BI__nvvm_atom_acq_rel_dec_global_ui:
21419 case NVPTX::BI__nvvm_atom_acq_rel_dec_global_ul:
21420 case NVPTX::BI__nvvm_atom_acq_rel_dec_global_ull:
21421 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_acq_rel, *
this, E);
21422 case NVPTX::BI__nvvm_atom_acq_rel_and_global_i:
21423 case NVPTX::BI__nvvm_atom_acq_rel_and_global_l:
21424 case NVPTX::BI__nvvm_atom_acq_rel_and_global_ll:
21425 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_acq_rel, *
this, E);
21426 case NVPTX::BI__nvvm_atom_acq_rel_or_global_i:
21427 case NVPTX::BI__nvvm_atom_acq_rel_or_global_l:
21428 case NVPTX::BI__nvvm_atom_acq_rel_or_global_ll:
21429 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_acq_rel, *
this, E);
21430 case NVPTX::BI__nvvm_atom_acq_rel_xor_global_i:
21431 case NVPTX::BI__nvvm_atom_acq_rel_xor_global_l:
21432 case NVPTX::BI__nvvm_atom_acq_rel_xor_global_ll:
21433 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_acq_rel, *
this, E);
21434 case NVPTX::BI__nvvm_atom_acq_rel_cas_global_i:
21435 case NVPTX::BI__nvvm_atom_acq_rel_cas_global_l:
21436 case NVPTX::BI__nvvm_atom_acq_rel_cas_global_ll:
21437 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_acq_rel);
21438 case NVPTX::BI__nvvm_atom_acq_rel_cas_global_f:
21439 case NVPTX::BI__nvvm_atom_acq_rel_cas_global_d:
21440 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_acq_rel);
21441 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_global_i:
21442 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_global_l:
21443 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_global_ll:
21444 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_acq_rel_cta, *
this, E);
21445 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_global_i:
21446 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_global_l:
21447 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_global_ll:
21448 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_i_acq_rel_sys, *
this, E);
21449 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_global_f:
21450 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_global_d:
21451 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_acq_rel_cta, *
this, E);
21452 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_global_f:
21453 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_global_d:
21454 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_global_f_acq_rel_sys, *
this, E);
21455 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_global_i:
21456 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_global_l:
21457 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_global_ll:
21458 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_acq_rel_cta, *
this, E);
21459 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_global_f:
21460 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_global_d:
21461 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_acq_rel_cta, *
this, E);
21462 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_global_i:
21463 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_global_l:
21464 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_global_ll:
21465 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_i_acq_rel_sys, *
this, E);
21466 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_global_f:
21467 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_global_d:
21468 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_global_f_acq_rel_sys, *
this, E);
21469 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_global_i:
21470 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_global_l:
21471 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_global_ll:
21472 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_acq_rel_cta, *
this, E);
21473 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_global_ui:
21474 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_global_ul:
21475 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_global_ull:
21476 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_acq_rel_cta, *
this, E);
21477 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_global_i:
21478 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_global_l:
21479 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_global_ll:
21480 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_i_acq_rel_sys, *
this, E);
21481 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_global_ui:
21482 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_global_ul:
21483 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_global_ull:
21484 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_global_ui_acq_rel_sys, *
this, E);
21485 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_global_i:
21486 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_global_l:
21487 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_global_ll:
21488 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_acq_rel_cta, *
this, E);
21489 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_global_ui:
21490 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_global_ul:
21491 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_global_ull:
21492 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_acq_rel_cta, *
this, E);
21493 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_global_i:
21494 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_global_l:
21495 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_global_ll:
21496 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_i_acq_rel_sys, *
this, E);
21497 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_global_ui:
21498 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_global_ul:
21499 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_global_ull:
21500 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_global_ui_acq_rel_sys, *
this, E);
21501 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_global_ui:
21502 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_global_ul:
21503 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_global_ull:
21504 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_acq_rel_cta, *
this, E);
21505 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_global_ui:
21506 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_global_ul:
21507 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_global_ull:
21508 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_acq_rel_cta, *
this, E);
21509 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_global_ui:
21510 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_global_ul:
21511 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_global_ull:
21512 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_global_i_acq_rel_sys, *
this, E);
21513 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_global_ui:
21514 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_global_ul:
21515 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_global_ull:
21516 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_global_i_acq_rel_sys, *
this, E);
21517 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_global_i:
21518 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_global_l:
21519 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_global_ll:
21520 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_acq_rel_cta, *
this, E);
21521 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_global_i:
21522 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_global_l:
21523 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_global_ll:
21524 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_global_i_acq_rel_sys, *
this, E);
21525 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_global_i:
21526 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_global_l:
21527 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_global_ll:
21528 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_acq_rel_cta, *
this, E);
21529 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_global_i:
21530 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_global_l:
21531 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_global_ll:
21532 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_global_i_acq_rel_sys, *
this, E);
21533 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_global_i:
21534 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_global_l:
21535 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_global_ll:
21536 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_acq_rel_cta, *
this, E);
21537 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_global_i:
21538 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_global_l:
21539 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_global_ll:
21540 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_global_i_acq_rel_sys, *
this, E);
21541 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_global_i:
21542 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_global_l:
21543 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_global_ll:
21544 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_acq_rel_cta);
21545 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_global_i:
21546 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_global_l:
21547 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_global_ll:
21548 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_i_acq_rel_sys);
21549 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_global_f:
21550 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_global_d:
21551 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_acq_rel_cta);
21552 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_global_f:
21553 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_global_d:
21554 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_global_f_acq_rel_sys);
21555 case NVPTX::BI__nvvm_atom_add_shared_i:
21556 case NVPTX::BI__nvvm_atom_add_shared_l:
21557 case NVPTX::BI__nvvm_atom_add_shared_ll:
21558 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i, *
this, E);
21559 case NVPTX::BI__nvvm_atom_add_shared_f:
21560 case NVPTX::BI__nvvm_atom_add_shared_d:
21561 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f, *
this, E);
21562 case NVPTX::BI__nvvm_atom_xchg_shared_i:
21563 case NVPTX::BI__nvvm_atom_xchg_shared_l:
21564 case NVPTX::BI__nvvm_atom_xchg_shared_ll:
21565 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i, *
this, E);
21566 case NVPTX::BI__nvvm_atom_xchg_shared_f:
21567 case NVPTX::BI__nvvm_atom_xchg_shared_d:
21568 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f, *
this, E);
21569 case NVPTX::BI__nvvm_atom_max_shared_i:
21570 case NVPTX::BI__nvvm_atom_max_shared_l:
21571 case NVPTX::BI__nvvm_atom_max_shared_ll:
21572 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i, *
this, E);
21573 case NVPTX::BI__nvvm_atom_max_shared_ui:
21574 case NVPTX::BI__nvvm_atom_max_shared_ul:
21575 case NVPTX::BI__nvvm_atom_max_shared_ull:
21576 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui, *
this, E);
21577 case NVPTX::BI__nvvm_atom_min_shared_i:
21578 case NVPTX::BI__nvvm_atom_min_shared_l:
21579 case NVPTX::BI__nvvm_atom_min_shared_ll:
21580 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i, *
this, E);
21581 case NVPTX::BI__nvvm_atom_min_shared_ui:
21582 case NVPTX::BI__nvvm_atom_min_shared_ul:
21583 case NVPTX::BI__nvvm_atom_min_shared_ull:
21584 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui, *
this, E);
21585 case NVPTX::BI__nvvm_atom_inc_shared_ui:
21586 case NVPTX::BI__nvvm_atom_inc_shared_ul:
21587 case NVPTX::BI__nvvm_atom_inc_shared_ull:
21588 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i, *
this, E);
21589 case NVPTX::BI__nvvm_atom_dec_shared_ui:
21590 case NVPTX::BI__nvvm_atom_dec_shared_ul:
21591 case NVPTX::BI__nvvm_atom_dec_shared_ull:
21592 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i, *
this, E);
21593 case NVPTX::BI__nvvm_atom_and_shared_i:
21594 case NVPTX::BI__nvvm_atom_and_shared_l:
21595 case NVPTX::BI__nvvm_atom_and_shared_ll:
21596 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i, *
this, E);
21597 case NVPTX::BI__nvvm_atom_or_shared_i:
21598 case NVPTX::BI__nvvm_atom_or_shared_l:
21599 case NVPTX::BI__nvvm_atom_or_shared_ll:
21600 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i, *
this, E);
21601 case NVPTX::BI__nvvm_atom_xor_shared_i:
21602 case NVPTX::BI__nvvm_atom_xor_shared_l:
21603 case NVPTX::BI__nvvm_atom_xor_shared_ll:
21604 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i, *
this, E);
21605 case NVPTX::BI__nvvm_atom_cas_shared_i:
21606 case NVPTX::BI__nvvm_atom_cas_shared_l:
21607 case NVPTX::BI__nvvm_atom_cas_shared_ll:
21608 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i);
21609 case NVPTX::BI__nvvm_atom_cas_shared_f:
21610 case NVPTX::BI__nvvm_atom_cas_shared_d:
21611 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f);
21612 case NVPTX::BI__nvvm_atom_cta_add_shared_i:
21613 case NVPTX::BI__nvvm_atom_cta_add_shared_l:
21614 case NVPTX::BI__nvvm_atom_cta_add_shared_ll:
21615 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_cta, *
this, E);
21616 case NVPTX::BI__nvvm_atom_sys_add_shared_i:
21617 case NVPTX::BI__nvvm_atom_sys_add_shared_l:
21618 case NVPTX::BI__nvvm_atom_sys_add_shared_ll:
21619 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_sys, *
this, E);
21620 case NVPTX::BI__nvvm_atom_cta_add_shared_f:
21621 case NVPTX::BI__nvvm_atom_cta_add_shared_d:
21622 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_cta, *
this, E);
21623 case NVPTX::BI__nvvm_atom_sys_add_shared_f:
21624 case NVPTX::BI__nvvm_atom_sys_add_shared_d:
21625 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_sys, *
this, E);
21626 case NVPTX::BI__nvvm_atom_cta_xchg_shared_i:
21627 case NVPTX::BI__nvvm_atom_cta_xchg_shared_l:
21628 case NVPTX::BI__nvvm_atom_cta_xchg_shared_ll:
21629 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_cta, *
this, E);
21630 case NVPTX::BI__nvvm_atom_cta_xchg_shared_f:
21631 case NVPTX::BI__nvvm_atom_cta_xchg_shared_d:
21632 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_cta, *
this, E);
21633 case NVPTX::BI__nvvm_atom_sys_xchg_shared_i:
21634 case NVPTX::BI__nvvm_atom_sys_xchg_shared_l:
21635 case NVPTX::BI__nvvm_atom_sys_xchg_shared_ll:
21636 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_sys, *
this, E);
21637 case NVPTX::BI__nvvm_atom_sys_xchg_shared_f:
21638 case NVPTX::BI__nvvm_atom_sys_xchg_shared_d:
21639 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_sys, *
this, E);
21640 case NVPTX::BI__nvvm_atom_cta_max_shared_i:
21641 case NVPTX::BI__nvvm_atom_cta_max_shared_l:
21642 case NVPTX::BI__nvvm_atom_cta_max_shared_ll:
21643 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_cta, *
this, E);
21644 case NVPTX::BI__nvvm_atom_cta_max_shared_ui:
21645 case NVPTX::BI__nvvm_atom_cta_max_shared_ul:
21646 case NVPTX::BI__nvvm_atom_cta_max_shared_ull:
21647 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_cta, *
this, E);
21648 case NVPTX::BI__nvvm_atom_sys_max_shared_i:
21649 case NVPTX::BI__nvvm_atom_sys_max_shared_l:
21650 case NVPTX::BI__nvvm_atom_sys_max_shared_ll:
21651 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_sys, *
this, E);
21652 case NVPTX::BI__nvvm_atom_sys_max_shared_ui:
21653 case NVPTX::BI__nvvm_atom_sys_max_shared_ul:
21654 case NVPTX::BI__nvvm_atom_sys_max_shared_ull:
21655 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_sys, *
this, E);
21656 case NVPTX::BI__nvvm_atom_cta_min_shared_i:
21657 case NVPTX::BI__nvvm_atom_cta_min_shared_l:
21658 case NVPTX::BI__nvvm_atom_cta_min_shared_ll:
21659 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_cta, *
this, E);
21660 case NVPTX::BI__nvvm_atom_cta_min_shared_ui:
21661 case NVPTX::BI__nvvm_atom_cta_min_shared_ul:
21662 case NVPTX::BI__nvvm_atom_cta_min_shared_ull:
21663 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_cta, *
this, E);
21664 case NVPTX::BI__nvvm_atom_sys_min_shared_i:
21665 case NVPTX::BI__nvvm_atom_sys_min_shared_l:
21666 case NVPTX::BI__nvvm_atom_sys_min_shared_ll:
21667 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_sys, *
this, E);
21668 case NVPTX::BI__nvvm_atom_sys_min_shared_ui:
21669 case NVPTX::BI__nvvm_atom_sys_min_shared_ul:
21670 case NVPTX::BI__nvvm_atom_sys_min_shared_ull:
21671 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_sys, *
this, E);
21672 case NVPTX::BI__nvvm_atom_cta_inc_shared_ui:
21673 case NVPTX::BI__nvvm_atom_cta_inc_shared_ul:
21674 case NVPTX::BI__nvvm_atom_cta_inc_shared_ull:
21675 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_cta, *
this, E);
21676 case NVPTX::BI__nvvm_atom_cta_dec_shared_ui:
21677 case NVPTX::BI__nvvm_atom_cta_dec_shared_ul:
21678 case NVPTX::BI__nvvm_atom_cta_dec_shared_ull:
21679 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_cta, *
this, E);
21680 case NVPTX::BI__nvvm_atom_sys_inc_shared_ui:
21681 case NVPTX::BI__nvvm_atom_sys_inc_shared_ul:
21682 case NVPTX::BI__nvvm_atom_sys_inc_shared_ull:
21683 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_sys, *
this, E);
21684 case NVPTX::BI__nvvm_atom_sys_dec_shared_ui:
21685 case NVPTX::BI__nvvm_atom_sys_dec_shared_ul:
21686 case NVPTX::BI__nvvm_atom_sys_dec_shared_ull:
21687 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_sys, *
this, E);
21688 case NVPTX::BI__nvvm_atom_cta_and_shared_i:
21689 case NVPTX::BI__nvvm_atom_cta_and_shared_l:
21690 case NVPTX::BI__nvvm_atom_cta_and_shared_ll:
21691 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_cta, *
this, E);
21692 case NVPTX::BI__nvvm_atom_sys_and_shared_i:
21693 case NVPTX::BI__nvvm_atom_sys_and_shared_l:
21694 case NVPTX::BI__nvvm_atom_sys_and_shared_ll:
21695 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_sys, *
this, E);
21696 case NVPTX::BI__nvvm_atom_cta_or_shared_i:
21697 case NVPTX::BI__nvvm_atom_cta_or_shared_l:
21698 case NVPTX::BI__nvvm_atom_cta_or_shared_ll:
21699 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_cta, *
this, E);
21700 case NVPTX::BI__nvvm_atom_sys_or_shared_i:
21701 case NVPTX::BI__nvvm_atom_sys_or_shared_l:
21702 case NVPTX::BI__nvvm_atom_sys_or_shared_ll:
21703 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_sys, *
this, E);
21704 case NVPTX::BI__nvvm_atom_cta_xor_shared_i:
21705 case NVPTX::BI__nvvm_atom_cta_xor_shared_l:
21706 case NVPTX::BI__nvvm_atom_cta_xor_shared_ll:
21707 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_cta, *
this, E);
21708 case NVPTX::BI__nvvm_atom_sys_xor_shared_i:
21709 case NVPTX::BI__nvvm_atom_sys_xor_shared_l:
21710 case NVPTX::BI__nvvm_atom_sys_xor_shared_ll:
21711 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_sys, *
this, E);
21712 case NVPTX::BI__nvvm_atom_cta_cas_shared_i:
21713 case NVPTX::BI__nvvm_atom_cta_cas_shared_l:
21714 case NVPTX::BI__nvvm_atom_cta_cas_shared_ll:
21715 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_cta);
21716 case NVPTX::BI__nvvm_atom_sys_cas_shared_i:
21717 case NVPTX::BI__nvvm_atom_sys_cas_shared_l:
21718 case NVPTX::BI__nvvm_atom_sys_cas_shared_ll:
21719 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_sys);
21720 case NVPTX::BI__nvvm_atom_cta_cas_shared_f:
21721 case NVPTX::BI__nvvm_atom_cta_cas_shared_d:
21722 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_cta);
21723 case NVPTX::BI__nvvm_atom_sys_cas_shared_f:
21724 case NVPTX::BI__nvvm_atom_sys_cas_shared_d:
21725 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_sys);
21726 case NVPTX::BI__nvvm_atom_acquire_add_shared_i:
21727 case NVPTX::BI__nvvm_atom_acquire_add_shared_l:
21728 case NVPTX::BI__nvvm_atom_acquire_add_shared_ll:
21729 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_acquire, *
this, E);
21730 case NVPTX::BI__nvvm_atom_acquire_add_shared_f:
21731 case NVPTX::BI__nvvm_atom_acquire_add_shared_d:
21732 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_acquire, *
this, E);
21733 case NVPTX::BI__nvvm_atom_acquire_xchg_shared_i:
21734 case NVPTX::BI__nvvm_atom_acquire_xchg_shared_l:
21735 case NVPTX::BI__nvvm_atom_acquire_xchg_shared_ll:
21736 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_acquire, *
this, E);
21737 case NVPTX::BI__nvvm_atom_acquire_xchg_shared_f:
21738 case NVPTX::BI__nvvm_atom_acquire_xchg_shared_d:
21739 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_acquire, *
this, E);
21740 case NVPTX::BI__nvvm_atom_acquire_max_shared_i:
21741 case NVPTX::BI__nvvm_atom_acquire_max_shared_l:
21742 case NVPTX::BI__nvvm_atom_acquire_max_shared_ll:
21743 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_acquire, *
this, E);
21744 case NVPTX::BI__nvvm_atom_acquire_max_shared_ui:
21745 case NVPTX::BI__nvvm_atom_acquire_max_shared_ul:
21746 case NVPTX::BI__nvvm_atom_acquire_max_shared_ull:
21747 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_acquire, *
this, E);
21748 case NVPTX::BI__nvvm_atom_acquire_min_shared_i:
21749 case NVPTX::BI__nvvm_atom_acquire_min_shared_l:
21750 case NVPTX::BI__nvvm_atom_acquire_min_shared_ll:
21751 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_acquire, *
this, E);
21752 case NVPTX::BI__nvvm_atom_acquire_min_shared_ui:
21753 case NVPTX::BI__nvvm_atom_acquire_min_shared_ul:
21754 case NVPTX::BI__nvvm_atom_acquire_min_shared_ull:
21755 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_acquire, *
this, E);
21756 case NVPTX::BI__nvvm_atom_acquire_inc_shared_ui:
21757 case NVPTX::BI__nvvm_atom_acquire_inc_shared_ul:
21758 case NVPTX::BI__nvvm_atom_acquire_inc_shared_ull:
21759 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_acquire, *
this, E);
21760 case NVPTX::BI__nvvm_atom_acquire_dec_shared_ui:
21761 case NVPTX::BI__nvvm_atom_acquire_dec_shared_ul:
21762 case NVPTX::BI__nvvm_atom_acquire_dec_shared_ull:
21763 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_acquire, *
this, E);
21764 case NVPTX::BI__nvvm_atom_acquire_and_shared_i:
21765 case NVPTX::BI__nvvm_atom_acquire_and_shared_l:
21766 case NVPTX::BI__nvvm_atom_acquire_and_shared_ll:
21767 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_acquire, *
this, E);
21768 case NVPTX::BI__nvvm_atom_acquire_or_shared_i:
21769 case NVPTX::BI__nvvm_atom_acquire_or_shared_l:
21770 case NVPTX::BI__nvvm_atom_acquire_or_shared_ll:
21771 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_acquire, *
this, E);
21772 case NVPTX::BI__nvvm_atom_acquire_xor_shared_i:
21773 case NVPTX::BI__nvvm_atom_acquire_xor_shared_l:
21774 case NVPTX::BI__nvvm_atom_acquire_xor_shared_ll:
21775 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_acquire, *
this, E);
21776 case NVPTX::BI__nvvm_atom_acquire_cas_shared_i:
21777 case NVPTX::BI__nvvm_atom_acquire_cas_shared_l:
21778 case NVPTX::BI__nvvm_atom_acquire_cas_shared_ll:
21779 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_acquire);
21780 case NVPTX::BI__nvvm_atom_acquire_cas_shared_f:
21781 case NVPTX::BI__nvvm_atom_acquire_cas_shared_d:
21782 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_acquire);
21783 case NVPTX::BI__nvvm_atom_acquire_cta_add_shared_i:
21784 case NVPTX::BI__nvvm_atom_acquire_cta_add_shared_l:
21785 case NVPTX::BI__nvvm_atom_acquire_cta_add_shared_ll:
21786 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_acquire_cta, *
this, E);
21787 case NVPTX::BI__nvvm_atom_acquire_sys_add_shared_i:
21788 case NVPTX::BI__nvvm_atom_acquire_sys_add_shared_l:
21789 case NVPTX::BI__nvvm_atom_acquire_sys_add_shared_ll:
21790 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_acquire_sys, *
this, E);
21791 case NVPTX::BI__nvvm_atom_acquire_cta_add_shared_f:
21792 case NVPTX::BI__nvvm_atom_acquire_cta_add_shared_d:
21793 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_acquire_cta, *
this, E);
21794 case NVPTX::BI__nvvm_atom_acquire_sys_add_shared_f:
21795 case NVPTX::BI__nvvm_atom_acquire_sys_add_shared_d:
21796 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_acquire_sys, *
this, E);
21797 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_shared_i:
21798 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_shared_l:
21799 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_shared_ll:
21800 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_acquire_cta, *
this, E);
21801 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_shared_f:
21802 case NVPTX::BI__nvvm_atom_acquire_cta_xchg_shared_d:
21803 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_acquire_cta, *
this, E);
21804 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_shared_i:
21805 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_shared_l:
21806 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_shared_ll:
21807 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_acquire_sys, *
this, E);
21808 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_shared_f:
21809 case NVPTX::BI__nvvm_atom_acquire_sys_xchg_shared_d:
21810 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_acquire_sys, *
this, E);
21811 case NVPTX::BI__nvvm_atom_acquire_cta_max_shared_i:
21812 case NVPTX::BI__nvvm_atom_acquire_cta_max_shared_l:
21813 case NVPTX::BI__nvvm_atom_acquire_cta_max_shared_ll:
21814 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_acquire_cta, *
this, E);
21815 case NVPTX::BI__nvvm_atom_acquire_cta_max_shared_ui:
21816 case NVPTX::BI__nvvm_atom_acquire_cta_max_shared_ul:
21817 case NVPTX::BI__nvvm_atom_acquire_cta_max_shared_ull:
21818 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_acquire_cta, *
this, E);
21819 case NVPTX::BI__nvvm_atom_acquire_sys_max_shared_i:
21820 case NVPTX::BI__nvvm_atom_acquire_sys_max_shared_l:
21821 case NVPTX::BI__nvvm_atom_acquire_sys_max_shared_ll:
21822 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_acquire_sys, *
this, E);
21823 case NVPTX::BI__nvvm_atom_acquire_sys_max_shared_ui:
21824 case NVPTX::BI__nvvm_atom_acquire_sys_max_shared_ul:
21825 case NVPTX::BI__nvvm_atom_acquire_sys_max_shared_ull:
21826 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_acquire_sys, *
this, E);
21827 case NVPTX::BI__nvvm_atom_acquire_cta_min_shared_i:
21828 case NVPTX::BI__nvvm_atom_acquire_cta_min_shared_l:
21829 case NVPTX::BI__nvvm_atom_acquire_cta_min_shared_ll:
21830 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_acquire_cta, *
this, E);
21831 case NVPTX::BI__nvvm_atom_acquire_cta_min_shared_ui:
21832 case NVPTX::BI__nvvm_atom_acquire_cta_min_shared_ul:
21833 case NVPTX::BI__nvvm_atom_acquire_cta_min_shared_ull:
21834 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_acquire_cta, *
this, E);
21835 case NVPTX::BI__nvvm_atom_acquire_sys_min_shared_i:
21836 case NVPTX::BI__nvvm_atom_acquire_sys_min_shared_l:
21837 case NVPTX::BI__nvvm_atom_acquire_sys_min_shared_ll:
21838 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_acquire_sys, *
this, E);
21839 case NVPTX::BI__nvvm_atom_acquire_sys_min_shared_ui:
21840 case NVPTX::BI__nvvm_atom_acquire_sys_min_shared_ul:
21841 case NVPTX::BI__nvvm_atom_acquire_sys_min_shared_ull:
21842 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_acquire_sys, *
this, E);
21843 case NVPTX::BI__nvvm_atom_acquire_cta_inc_shared_ui:
21844 case NVPTX::BI__nvvm_atom_acquire_cta_inc_shared_ul:
21845 case NVPTX::BI__nvvm_atom_acquire_cta_inc_shared_ull:
21846 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_acquire_cta, *
this, E);
21847 case NVPTX::BI__nvvm_atom_acquire_cta_dec_shared_ui:
21848 case NVPTX::BI__nvvm_atom_acquire_cta_dec_shared_ul:
21849 case NVPTX::BI__nvvm_atom_acquire_cta_dec_shared_ull:
21850 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_acquire_cta, *
this, E);
21851 case NVPTX::BI__nvvm_atom_acquire_sys_inc_shared_ui:
21852 case NVPTX::BI__nvvm_atom_acquire_sys_inc_shared_ul:
21853 case NVPTX::BI__nvvm_atom_acquire_sys_inc_shared_ull:
21854 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_acquire_sys, *
this, E);
21855 case NVPTX::BI__nvvm_atom_acquire_sys_dec_shared_ui:
21856 case NVPTX::BI__nvvm_atom_acquire_sys_dec_shared_ul:
21857 case NVPTX::BI__nvvm_atom_acquire_sys_dec_shared_ull:
21858 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_acquire_sys, *
this, E);
21859 case NVPTX::BI__nvvm_atom_acquire_cta_and_shared_i:
21860 case NVPTX::BI__nvvm_atom_acquire_cta_and_shared_l:
21861 case NVPTX::BI__nvvm_atom_acquire_cta_and_shared_ll:
21862 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_acquire_cta, *
this, E);
21863 case NVPTX::BI__nvvm_atom_acquire_sys_and_shared_i:
21864 case NVPTX::BI__nvvm_atom_acquire_sys_and_shared_l:
21865 case NVPTX::BI__nvvm_atom_acquire_sys_and_shared_ll:
21866 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_acquire_sys, *
this, E);
21867 case NVPTX::BI__nvvm_atom_acquire_cta_or_shared_i:
21868 case NVPTX::BI__nvvm_atom_acquire_cta_or_shared_l:
21869 case NVPTX::BI__nvvm_atom_acquire_cta_or_shared_ll:
21870 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_acquire_cta, *
this, E);
21871 case NVPTX::BI__nvvm_atom_acquire_sys_or_shared_i:
21872 case NVPTX::BI__nvvm_atom_acquire_sys_or_shared_l:
21873 case NVPTX::BI__nvvm_atom_acquire_sys_or_shared_ll:
21874 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_acquire_sys, *
this, E);
21875 case NVPTX::BI__nvvm_atom_acquire_cta_xor_shared_i:
21876 case NVPTX::BI__nvvm_atom_acquire_cta_xor_shared_l:
21877 case NVPTX::BI__nvvm_atom_acquire_cta_xor_shared_ll:
21878 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_acquire_cta, *
this, E);
21879 case NVPTX::BI__nvvm_atom_acquire_sys_xor_shared_i:
21880 case NVPTX::BI__nvvm_atom_acquire_sys_xor_shared_l:
21881 case NVPTX::BI__nvvm_atom_acquire_sys_xor_shared_ll:
21882 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_acquire_sys, *
this, E);
21883 case NVPTX::BI__nvvm_atom_acquire_cta_cas_shared_i:
21884 case NVPTX::BI__nvvm_atom_acquire_cta_cas_shared_l:
21885 case NVPTX::BI__nvvm_atom_acquire_cta_cas_shared_ll:
21886 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_acquire_cta);
21887 case NVPTX::BI__nvvm_atom_acquire_sys_cas_shared_i:
21888 case NVPTX::BI__nvvm_atom_acquire_sys_cas_shared_l:
21889 case NVPTX::BI__nvvm_atom_acquire_sys_cas_shared_ll:
21890 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_acquire_sys);
21891 case NVPTX::BI__nvvm_atom_acquire_cta_cas_shared_f:
21892 case NVPTX::BI__nvvm_atom_acquire_cta_cas_shared_d:
21893 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_acquire_cta);
21894 case NVPTX::BI__nvvm_atom_acquire_sys_cas_shared_f:
21895 case NVPTX::BI__nvvm_atom_acquire_sys_cas_shared_d:
21896 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_acquire_sys);
21897 case NVPTX::BI__nvvm_atom_release_add_shared_i:
21898 case NVPTX::BI__nvvm_atom_release_add_shared_l:
21899 case NVPTX::BI__nvvm_atom_release_add_shared_ll:
21900 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_release, *
this, E);
21901 case NVPTX::BI__nvvm_atom_release_add_shared_f:
21902 case NVPTX::BI__nvvm_atom_release_add_shared_d:
21903 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_release, *
this, E);
21904 case NVPTX::BI__nvvm_atom_release_xchg_shared_i:
21905 case NVPTX::BI__nvvm_atom_release_xchg_shared_l:
21906 case NVPTX::BI__nvvm_atom_release_xchg_shared_ll:
21907 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_release, *
this, E);
21908 case NVPTX::BI__nvvm_atom_release_xchg_shared_f:
21909 case NVPTX::BI__nvvm_atom_release_xchg_shared_d:
21910 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_release, *
this, E);
21911 case NVPTX::BI__nvvm_atom_release_max_shared_i:
21912 case NVPTX::BI__nvvm_atom_release_max_shared_l:
21913 case NVPTX::BI__nvvm_atom_release_max_shared_ll:
21914 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_release, *
this, E);
21915 case NVPTX::BI__nvvm_atom_release_max_shared_ui:
21916 case NVPTX::BI__nvvm_atom_release_max_shared_ul:
21917 case NVPTX::BI__nvvm_atom_release_max_shared_ull:
21918 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_release, *
this, E);
21919 case NVPTX::BI__nvvm_atom_release_min_shared_i:
21920 case NVPTX::BI__nvvm_atom_release_min_shared_l:
21921 case NVPTX::BI__nvvm_atom_release_min_shared_ll:
21922 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_release, *
this, E);
21923 case NVPTX::BI__nvvm_atom_release_min_shared_ui:
21924 case NVPTX::BI__nvvm_atom_release_min_shared_ul:
21925 case NVPTX::BI__nvvm_atom_release_min_shared_ull:
21926 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_release, *
this, E);
21927 case NVPTX::BI__nvvm_atom_release_inc_shared_ui:
21928 case NVPTX::BI__nvvm_atom_release_inc_shared_ul:
21929 case NVPTX::BI__nvvm_atom_release_inc_shared_ull:
21930 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_release, *
this, E);
21931 case NVPTX::BI__nvvm_atom_release_dec_shared_ui:
21932 case NVPTX::BI__nvvm_atom_release_dec_shared_ul:
21933 case NVPTX::BI__nvvm_atom_release_dec_shared_ull:
21934 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_release, *
this, E);
21935 case NVPTX::BI__nvvm_atom_release_and_shared_i:
21936 case NVPTX::BI__nvvm_atom_release_and_shared_l:
21937 case NVPTX::BI__nvvm_atom_release_and_shared_ll:
21938 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_release, *
this, E);
21939 case NVPTX::BI__nvvm_atom_release_or_shared_i:
21940 case NVPTX::BI__nvvm_atom_release_or_shared_l:
21941 case NVPTX::BI__nvvm_atom_release_or_shared_ll:
21942 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_release, *
this, E);
21943 case NVPTX::BI__nvvm_atom_release_xor_shared_i:
21944 case NVPTX::BI__nvvm_atom_release_xor_shared_l:
21945 case NVPTX::BI__nvvm_atom_release_xor_shared_ll:
21946 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_release, *
this, E);
21947 case NVPTX::BI__nvvm_atom_release_cas_shared_i:
21948 case NVPTX::BI__nvvm_atom_release_cas_shared_l:
21949 case NVPTX::BI__nvvm_atom_release_cas_shared_ll:
21950 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_release);
21951 case NVPTX::BI__nvvm_atom_release_cas_shared_f:
21952 case NVPTX::BI__nvvm_atom_release_cas_shared_d:
21953 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_release);
21954 case NVPTX::BI__nvvm_atom_release_cta_add_shared_i:
21955 case NVPTX::BI__nvvm_atom_release_cta_add_shared_l:
21956 case NVPTX::BI__nvvm_atom_release_cta_add_shared_ll:
21957 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_release_cta, *
this, E);
21958 case NVPTX::BI__nvvm_atom_release_sys_add_shared_i:
21959 case NVPTX::BI__nvvm_atom_release_sys_add_shared_l:
21960 case NVPTX::BI__nvvm_atom_release_sys_add_shared_ll:
21961 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_release_sys, *
this, E);
21962 case NVPTX::BI__nvvm_atom_release_cta_add_shared_f:
21963 case NVPTX::BI__nvvm_atom_release_cta_add_shared_d:
21964 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_release_cta, *
this, E);
21965 case NVPTX::BI__nvvm_atom_release_sys_add_shared_f:
21966 case NVPTX::BI__nvvm_atom_release_sys_add_shared_d:
21967 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_release_sys, *
this, E);
21968 case NVPTX::BI__nvvm_atom_release_cta_xchg_shared_i:
21969 case NVPTX::BI__nvvm_atom_release_cta_xchg_shared_l:
21970 case NVPTX::BI__nvvm_atom_release_cta_xchg_shared_ll:
21971 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_release_cta, *
this, E);
21972 case NVPTX::BI__nvvm_atom_release_cta_xchg_shared_f:
21973 case NVPTX::BI__nvvm_atom_release_cta_xchg_shared_d:
21974 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_release_cta, *
this, E);
21975 case NVPTX::BI__nvvm_atom_release_sys_xchg_shared_i:
21976 case NVPTX::BI__nvvm_atom_release_sys_xchg_shared_l:
21977 case NVPTX::BI__nvvm_atom_release_sys_xchg_shared_ll:
21978 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_release_sys, *
this, E);
21979 case NVPTX::BI__nvvm_atom_release_sys_xchg_shared_f:
21980 case NVPTX::BI__nvvm_atom_release_sys_xchg_shared_d:
21981 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_release_sys, *
this, E);
21982 case NVPTX::BI__nvvm_atom_release_cta_max_shared_i:
21983 case NVPTX::BI__nvvm_atom_release_cta_max_shared_l:
21984 case NVPTX::BI__nvvm_atom_release_cta_max_shared_ll:
21985 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_release_cta, *
this, E);
21986 case NVPTX::BI__nvvm_atom_release_cta_max_shared_ui:
21987 case NVPTX::BI__nvvm_atom_release_cta_max_shared_ul:
21988 case NVPTX::BI__nvvm_atom_release_cta_max_shared_ull:
21989 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_release_cta, *
this, E);
21990 case NVPTX::BI__nvvm_atom_release_sys_max_shared_i:
21991 case NVPTX::BI__nvvm_atom_release_sys_max_shared_l:
21992 case NVPTX::BI__nvvm_atom_release_sys_max_shared_ll:
21993 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_release_sys, *
this, E);
21994 case NVPTX::BI__nvvm_atom_release_sys_max_shared_ui:
21995 case NVPTX::BI__nvvm_atom_release_sys_max_shared_ul:
21996 case NVPTX::BI__nvvm_atom_release_sys_max_shared_ull:
21997 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_release_sys, *
this, E);
21998 case NVPTX::BI__nvvm_atom_release_cta_min_shared_i:
21999 case NVPTX::BI__nvvm_atom_release_cta_min_shared_l:
22000 case NVPTX::BI__nvvm_atom_release_cta_min_shared_ll:
22001 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_release_cta, *
this, E);
22002 case NVPTX::BI__nvvm_atom_release_cta_min_shared_ui:
22003 case NVPTX::BI__nvvm_atom_release_cta_min_shared_ul:
22004 case NVPTX::BI__nvvm_atom_release_cta_min_shared_ull:
22005 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_release_cta, *
this, E);
22006 case NVPTX::BI__nvvm_atom_release_sys_min_shared_i:
22007 case NVPTX::BI__nvvm_atom_release_sys_min_shared_l:
22008 case NVPTX::BI__nvvm_atom_release_sys_min_shared_ll:
22009 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_release_sys, *
this, E);
22010 case NVPTX::BI__nvvm_atom_release_sys_min_shared_ui:
22011 case NVPTX::BI__nvvm_atom_release_sys_min_shared_ul:
22012 case NVPTX::BI__nvvm_atom_release_sys_min_shared_ull:
22013 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_release_sys, *
this, E);
22014 case NVPTX::BI__nvvm_atom_release_cta_inc_shared_ui:
22015 case NVPTX::BI__nvvm_atom_release_cta_inc_shared_ul:
22016 case NVPTX::BI__nvvm_atom_release_cta_inc_shared_ull:
22017 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_release_cta, *
this, E);
22018 case NVPTX::BI__nvvm_atom_release_cta_dec_shared_ui:
22019 case NVPTX::BI__nvvm_atom_release_cta_dec_shared_ul:
22020 case NVPTX::BI__nvvm_atom_release_cta_dec_shared_ull:
22021 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_release_cta, *
this, E);
22022 case NVPTX::BI__nvvm_atom_release_sys_inc_shared_ui:
22023 case NVPTX::BI__nvvm_atom_release_sys_inc_shared_ul:
22024 case NVPTX::BI__nvvm_atom_release_sys_inc_shared_ull:
22025 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_release_sys, *
this, E);
22026 case NVPTX::BI__nvvm_atom_release_sys_dec_shared_ui:
22027 case NVPTX::BI__nvvm_atom_release_sys_dec_shared_ul:
22028 case NVPTX::BI__nvvm_atom_release_sys_dec_shared_ull:
22029 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_release_sys, *
this, E);
22030 case NVPTX::BI__nvvm_atom_release_cta_and_shared_i:
22031 case NVPTX::BI__nvvm_atom_release_cta_and_shared_l:
22032 case NVPTX::BI__nvvm_atom_release_cta_and_shared_ll:
22033 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_release_cta, *
this, E);
22034 case NVPTX::BI__nvvm_atom_release_sys_and_shared_i:
22035 case NVPTX::BI__nvvm_atom_release_sys_and_shared_l:
22036 case NVPTX::BI__nvvm_atom_release_sys_and_shared_ll:
22037 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_release_sys, *
this, E);
22038 case NVPTX::BI__nvvm_atom_release_cta_or_shared_i:
22039 case NVPTX::BI__nvvm_atom_release_cta_or_shared_l:
22040 case NVPTX::BI__nvvm_atom_release_cta_or_shared_ll:
22041 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_release_cta, *
this, E);
22042 case NVPTX::BI__nvvm_atom_release_sys_or_shared_i:
22043 case NVPTX::BI__nvvm_atom_release_sys_or_shared_l:
22044 case NVPTX::BI__nvvm_atom_release_sys_or_shared_ll:
22045 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_release_sys, *
this, E);
22046 case NVPTX::BI__nvvm_atom_release_cta_xor_shared_i:
22047 case NVPTX::BI__nvvm_atom_release_cta_xor_shared_l:
22048 case NVPTX::BI__nvvm_atom_release_cta_xor_shared_ll:
22049 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_release_cta, *
this, E);
22050 case NVPTX::BI__nvvm_atom_release_sys_xor_shared_i:
22051 case NVPTX::BI__nvvm_atom_release_sys_xor_shared_l:
22052 case NVPTX::BI__nvvm_atom_release_sys_xor_shared_ll:
22053 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_release_sys, *
this, E);
22054 case NVPTX::BI__nvvm_atom_release_cta_cas_shared_i:
22055 case NVPTX::BI__nvvm_atom_release_cta_cas_shared_l:
22056 case NVPTX::BI__nvvm_atom_release_cta_cas_shared_ll:
22057 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_release_cta);
22058 case NVPTX::BI__nvvm_atom_release_sys_cas_shared_i:
22059 case NVPTX::BI__nvvm_atom_release_sys_cas_shared_l:
22060 case NVPTX::BI__nvvm_atom_release_sys_cas_shared_ll:
22061 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_release_sys);
22062 case NVPTX::BI__nvvm_atom_release_cta_cas_shared_f:
22063 case NVPTX::BI__nvvm_atom_release_cta_cas_shared_d:
22064 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_release_cta);
22065 case NVPTX::BI__nvvm_atom_release_sys_cas_shared_f:
22066 case NVPTX::BI__nvvm_atom_release_sys_cas_shared_d:
22067 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_release_sys);
22068 case NVPTX::BI__nvvm_atom_acq_rel_add_shared_i:
22069 case NVPTX::BI__nvvm_atom_acq_rel_add_shared_l:
22070 case NVPTX::BI__nvvm_atom_acq_rel_add_shared_ll:
22071 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_acq_rel, *
this, E);
22072 case NVPTX::BI__nvvm_atom_acq_rel_add_shared_f:
22073 case NVPTX::BI__nvvm_atom_acq_rel_add_shared_d:
22074 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_acq_rel, *
this, E);
22075 case NVPTX::BI__nvvm_atom_acq_rel_xchg_shared_i:
22076 case NVPTX::BI__nvvm_atom_acq_rel_xchg_shared_l:
22077 case NVPTX::BI__nvvm_atom_acq_rel_xchg_shared_ll:
22078 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_acq_rel, *
this, E);
22079 case NVPTX::BI__nvvm_atom_acq_rel_xchg_shared_f:
22080 case NVPTX::BI__nvvm_atom_acq_rel_xchg_shared_d:
22081 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_acq_rel, *
this, E);
22082 case NVPTX::BI__nvvm_atom_acq_rel_max_shared_i:
22083 case NVPTX::BI__nvvm_atom_acq_rel_max_shared_l:
22084 case NVPTX::BI__nvvm_atom_acq_rel_max_shared_ll:
22085 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_acq_rel, *
this, E);
22086 case NVPTX::BI__nvvm_atom_acq_rel_max_shared_ui:
22087 case NVPTX::BI__nvvm_atom_acq_rel_max_shared_ul:
22088 case NVPTX::BI__nvvm_atom_acq_rel_max_shared_ull:
22089 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_acq_rel, *
this, E);
22090 case NVPTX::BI__nvvm_atom_acq_rel_min_shared_i:
22091 case NVPTX::BI__nvvm_atom_acq_rel_min_shared_l:
22092 case NVPTX::BI__nvvm_atom_acq_rel_min_shared_ll:
22093 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_acq_rel, *
this, E);
22094 case NVPTX::BI__nvvm_atom_acq_rel_min_shared_ui:
22095 case NVPTX::BI__nvvm_atom_acq_rel_min_shared_ul:
22096 case NVPTX::BI__nvvm_atom_acq_rel_min_shared_ull:
22097 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_acq_rel, *
this, E);
22098 case NVPTX::BI__nvvm_atom_acq_rel_inc_shared_ui:
22099 case NVPTX::BI__nvvm_atom_acq_rel_inc_shared_ul:
22100 case NVPTX::BI__nvvm_atom_acq_rel_inc_shared_ull:
22101 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_acq_rel, *
this, E);
22102 case NVPTX::BI__nvvm_atom_acq_rel_dec_shared_ui:
22103 case NVPTX::BI__nvvm_atom_acq_rel_dec_shared_ul:
22104 case NVPTX::BI__nvvm_atom_acq_rel_dec_shared_ull:
22105 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_acq_rel, *
this, E);
22106 case NVPTX::BI__nvvm_atom_acq_rel_and_shared_i:
22107 case NVPTX::BI__nvvm_atom_acq_rel_and_shared_l:
22108 case NVPTX::BI__nvvm_atom_acq_rel_and_shared_ll:
22109 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_acq_rel, *
this, E);
22110 case NVPTX::BI__nvvm_atom_acq_rel_or_shared_i:
22111 case NVPTX::BI__nvvm_atom_acq_rel_or_shared_l:
22112 case NVPTX::BI__nvvm_atom_acq_rel_or_shared_ll:
22113 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_acq_rel, *
this, E);
22114 case NVPTX::BI__nvvm_atom_acq_rel_xor_shared_i:
22115 case NVPTX::BI__nvvm_atom_acq_rel_xor_shared_l:
22116 case NVPTX::BI__nvvm_atom_acq_rel_xor_shared_ll:
22117 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_acq_rel, *
this, E);
22118 case NVPTX::BI__nvvm_atom_acq_rel_cas_shared_i:
22119 case NVPTX::BI__nvvm_atom_acq_rel_cas_shared_l:
22120 case NVPTX::BI__nvvm_atom_acq_rel_cas_shared_ll:
22121 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_acq_rel);
22122 case NVPTX::BI__nvvm_atom_acq_rel_cas_shared_f:
22123 case NVPTX::BI__nvvm_atom_acq_rel_cas_shared_d:
22124 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_acq_rel);
22125 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_shared_i:
22126 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_shared_l:
22127 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_shared_ll:
22128 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_acq_rel_cta, *
this, E);
22129 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_shared_i:
22130 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_shared_l:
22131 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_shared_ll:
22132 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_i_acq_rel_sys, *
this, E);
22133 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_shared_f:
22134 case NVPTX::BI__nvvm_atom_acq_rel_cta_add_shared_d:
22135 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_acq_rel_cta, *
this, E);
22136 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_shared_f:
22137 case NVPTX::BI__nvvm_atom_acq_rel_sys_add_shared_d:
22138 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_shared_f_acq_rel_sys, *
this, E);
22139 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_shared_i:
22140 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_shared_l:
22141 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_shared_ll:
22142 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_acq_rel_cta, *
this, E);
22143 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_shared_f:
22144 case NVPTX::BI__nvvm_atom_acq_rel_cta_xchg_shared_d:
22145 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_acq_rel_cta, *
this, E);
22146 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_shared_i:
22147 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_shared_l:
22148 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_shared_ll:
22149 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_i_acq_rel_sys, *
this, E);
22150 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_shared_f:
22151 case NVPTX::BI__nvvm_atom_acq_rel_sys_xchg_shared_d:
22152 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_shared_f_acq_rel_sys, *
this, E);
22153 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_shared_i:
22154 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_shared_l:
22155 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_shared_ll:
22156 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_acq_rel_cta, *
this, E);
22157 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_shared_ui:
22158 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_shared_ul:
22159 case NVPTX::BI__nvvm_atom_acq_rel_cta_max_shared_ull:
22160 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_acq_rel_cta, *
this, E);
22161 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_shared_i:
22162 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_shared_l:
22163 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_shared_ll:
22164 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_i_acq_rel_sys, *
this, E);
22165 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_shared_ui:
22166 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_shared_ul:
22167 case NVPTX::BI__nvvm_atom_acq_rel_sys_max_shared_ull:
22168 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_shared_ui_acq_rel_sys, *
this, E);
22169 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_shared_i:
22170 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_shared_l:
22171 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_shared_ll:
22172 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_acq_rel_cta, *
this, E);
22173 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_shared_ui:
22174 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_shared_ul:
22175 case NVPTX::BI__nvvm_atom_acq_rel_cta_min_shared_ull:
22176 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_acq_rel_cta, *
this, E);
22177 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_shared_i:
22178 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_shared_l:
22179 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_shared_ll:
22180 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_i_acq_rel_sys, *
this, E);
22181 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_shared_ui:
22182 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_shared_ul:
22183 case NVPTX::BI__nvvm_atom_acq_rel_sys_min_shared_ull:
22184 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_shared_ui_acq_rel_sys, *
this, E);
22185 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_shared_ui:
22186 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_shared_ul:
22187 case NVPTX::BI__nvvm_atom_acq_rel_cta_inc_shared_ull:
22188 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_acq_rel_cta, *
this, E);
22189 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_shared_ui:
22190 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_shared_ul:
22191 case NVPTX::BI__nvvm_atom_acq_rel_cta_dec_shared_ull:
22192 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_acq_rel_cta, *
this, E);
22193 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_shared_ui:
22194 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_shared_ul:
22195 case NVPTX::BI__nvvm_atom_acq_rel_sys_inc_shared_ull:
22196 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_shared_i_acq_rel_sys, *
this, E);
22197 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_shared_ui:
22198 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_shared_ul:
22199 case NVPTX::BI__nvvm_atom_acq_rel_sys_dec_shared_ull:
22200 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_shared_i_acq_rel_sys, *
this, E);
22201 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_shared_i:
22202 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_shared_l:
22203 case NVPTX::BI__nvvm_atom_acq_rel_cta_and_shared_ll:
22204 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_acq_rel_cta, *
this, E);
22205 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_shared_i:
22206 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_shared_l:
22207 case NVPTX::BI__nvvm_atom_acq_rel_sys_and_shared_ll:
22208 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_shared_i_acq_rel_sys, *
this, E);
22209 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_shared_i:
22210 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_shared_l:
22211 case NVPTX::BI__nvvm_atom_acq_rel_cta_or_shared_ll:
22212 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_acq_rel_cta, *
this, E);
22213 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_shared_i:
22214 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_shared_l:
22215 case NVPTX::BI__nvvm_atom_acq_rel_sys_or_shared_ll:
22216 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_shared_i_acq_rel_sys, *
this, E);
22217 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_shared_i:
22218 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_shared_l:
22219 case NVPTX::BI__nvvm_atom_acq_rel_cta_xor_shared_ll:
22220 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_acq_rel_cta, *
this, E);
22221 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_shared_i:
22222 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_shared_l:
22223 case NVPTX::BI__nvvm_atom_acq_rel_sys_xor_shared_ll:
22224 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_shared_i_acq_rel_sys, *
this, E);
22225 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_shared_i:
22226 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_shared_l:
22227 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_shared_ll:
22228 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_acq_rel_cta);
22229 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_shared_i:
22230 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_shared_l:
22231 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_shared_ll:
22232 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_i_acq_rel_sys);
22233 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_shared_f:
22234 case NVPTX::BI__nvvm_atom_acq_rel_cta_cas_shared_d:
22235 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_acq_rel_cta);
22236 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_shared_f:
22237 case NVPTX::BI__nvvm_atom_acq_rel_sys_cas_shared_d:
22238 return MakeScopedCasAtomic(Intrinsic::nvvm_atomic_cas_shared_f_acq_rel_sys);
22239 case NVPTX::BI__nvvm_match_all_sync_i32p:
22240 case NVPTX::BI__nvvm_match_all_sync_i64p: {
22244 Value *ResultPair = Builder.CreateCall(
22245 CGM.getIntrinsic(BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
22246 ? Intrinsic::nvvm_match_all_sync_i32p
22247 : Intrinsic::nvvm_match_all_sync_i64p),
22249 Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
22251 Builder.CreateStore(Pred, PredOutPtr);
22252 return Builder.CreateExtractValue(ResultPair, 0);
22256 case NVPTX::BI__hmma_m16n16k16_ld_a:
22257 case NVPTX::BI__hmma_m16n16k16_ld_b:
22258 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
22259 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
22260 case NVPTX::BI__hmma_m32n8k16_ld_a:
22261 case NVPTX::BI__hmma_m32n8k16_ld_b:
22262 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
22263 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
22264 case NVPTX::BI__hmma_m8n32k16_ld_a:
22265 case NVPTX::BI__hmma_m8n32k16_ld_b:
22266 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
22267 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
22269 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
22270 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
22271 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
22272 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
22273 case NVPTX::BI__imma_m16n16k16_ld_c:
22274 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
22275 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
22276 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
22277 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
22278 case NVPTX::BI__imma_m32n8k16_ld_c:
22279 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
22280 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
22281 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
22282 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
22283 case NVPTX::BI__imma_m8n32k16_ld_c:
22285 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
22286 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
22287 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
22288 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
22289 case NVPTX::BI__imma_m8n8k32_ld_c:
22290 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
22291 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
22292 case NVPTX::BI__bmma_m8n8k128_ld_c:
22294 case NVPTX::BI__dmma_m8n8k4_ld_a:
22295 case NVPTX::BI__dmma_m8n8k4_ld_b:
22296 case NVPTX::BI__dmma_m8n8k4_ld_c:
22298 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
22299 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
22300 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
22301 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
22302 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
22303 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
22304 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
22305 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
22306 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
22310 std::optional<llvm::APSInt> isColMajorArg =
22312 if (!isColMajorArg)
22314 bool isColMajor = isColMajorArg->getSExtValue();
22315 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
22316 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
22321 Builder.CreateCall(CGM.getIntrinsic(IID, Src->
getType()), {Src, Ldm});
22324 assert(II.NumResults);
22325 if (II.NumResults == 1) {
22329 for (
unsigned i = 0; i < II.NumResults; ++i) {
22330 Builder.CreateAlignedStore(
22331 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
22334 llvm::ConstantInt::get(IntTy, i)),
22341 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
22342 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
22343 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
22344 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
22345 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
22346 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
22347 case NVPTX::BI__imma_m16n16k16_st_c_i32:
22348 case NVPTX::BI__imma_m32n8k16_st_c_i32:
22349 case NVPTX::BI__imma_m8n32k16_st_c_i32:
22350 case NVPTX::BI__imma_m8n8k32_st_c_i32:
22351 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
22352 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
22353 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
22357 std::optional<llvm::APSInt> isColMajorArg =
22359 if (!isColMajorArg)
22361 bool isColMajor = isColMajorArg->getSExtValue();
22362 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
22363 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
22367 CGM.getIntrinsic(IID, Dst->
getType());
22368 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
22370 for (
unsigned i = 0; i < II.NumResults; ++i) {
22371 Value *
V = Builder.CreateAlignedLoad(
22374 llvm::ConstantInt::get(IntTy, i)),
22376 Values.push_back(Builder.CreateBitCast(
V, ParamType));
22378 Values.push_back(Ldm);
22379 Value *Result = Builder.CreateCall(Intrinsic, Values);
22385 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
22386 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
22387 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
22388 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
22389 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
22390 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
22391 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
22392 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
22393 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
22394 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
22395 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
22396 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
22397 case NVPTX::BI__imma_m16n16k16_mma_s8:
22398 case NVPTX::BI__imma_m16n16k16_mma_u8:
22399 case NVPTX::BI__imma_m32n8k16_mma_s8:
22400 case NVPTX::BI__imma_m32n8k16_mma_u8:
22401 case NVPTX::BI__imma_m8n32k16_mma_s8:
22402 case NVPTX::BI__imma_m8n32k16_mma_u8:
22403 case NVPTX::BI__imma_m8n8k32_mma_s4:
22404 case NVPTX::BI__imma_m8n8k32_mma_u4:
22405 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
22406 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
22407 case NVPTX::BI__dmma_m8n8k4_mma_f64:
22408 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
22409 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
22410 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
22411 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
22416 std::optional<llvm::APSInt> LayoutArg =
22420 int Layout = LayoutArg->getSExtValue();
22421 if (Layout < 0 || Layout > 3)
22424 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
22425 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
22427 else if (std::optional<llvm::APSInt> OptSatfArg =
22429 SatfArg = *OptSatfArg;
22432 bool Satf = SatfArg.getSExtValue();
22433 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
22434 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
22439 Function *Intrinsic = CGM.getIntrinsic(IID);
22440 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
22442 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
22443 Value *
V = Builder.CreateAlignedLoad(
22446 llvm::ConstantInt::get(IntTy, i)),
22448 Values.push_back(Builder.CreateBitCast(
V, AType));
22451 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
22452 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
22453 Value *
V = Builder.CreateAlignedLoad(
22456 llvm::ConstantInt::get(IntTy, i)),
22458 Values.push_back(Builder.CreateBitCast(
V, BType));
22461 llvm::Type *CType =
22462 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
22463 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
22464 Value *
V = Builder.CreateAlignedLoad(
22467 llvm::ConstantInt::get(IntTy, i)),
22469 Values.push_back(Builder.CreateBitCast(
V, CType));
22471 Value *Result = Builder.CreateCall(Intrinsic, Values);
22473 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
22474 Builder.CreateAlignedStore(
22475 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
DType),
22477 llvm::ConstantInt::get(IntTy, i)),
22482 case NVPTX::BI__nvvm_ex2_approx_f16:
22483 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID, E, *
this);
22484 case NVPTX::BI__nvvm_ex2_approx_f16x2:
22485 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID, E, *
this);
22486 case NVPTX::BI__nvvm_ff2f16x2_rn:
22487 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, *
this);
22488 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
22489 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, *
this);
22490 case NVPTX::BI__nvvm_ff2f16x2_rz:
22491 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, *
this);
22492 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
22493 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, *
this);
22494 case NVPTX::BI__nvvm_fma_rn_f16:
22495 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *
this);
22496 case NVPTX::BI__nvvm_fma_rn_f16x2:
22497 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *
this);
22498 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
22499 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *
this);
22500 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
22501 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *
this);
22502 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
22503 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
22505 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
22506 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
22508 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
22509 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
22511 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
22512 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
22514 case NVPTX::BI__nvvm_fma_rn_relu_f16:
22515 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *
this);
22516 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
22517 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *
this);
22518 case NVPTX::BI__nvvm_fma_rn_sat_f16:
22519 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *
this);
22520 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
22521 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *
this);
22522 case NVPTX::BI__nvvm_fmax_f16:
22523 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID, E, *
this);
22524 case NVPTX::BI__nvvm_fmax_f16x2:
22525 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, *
this);
22526 case NVPTX::BI__nvvm_fmax_ftz_f16:
22527 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, *
this);
22528 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
22529 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, *
this);
22530 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
22531 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, *
this);
22532 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
22533 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
22535 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
22536 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
22538 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
22539 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
22540 BuiltinID, E, *
this);
22541 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
22542 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
22544 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
22545 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
22547 case NVPTX::BI__nvvm_fmax_nan_f16:
22548 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, *
this);
22549 case NVPTX::BI__nvvm_fmax_nan_f16x2:
22550 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, *
this);
22551 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
22552 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
22554 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
22555 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
22557 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
22558 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
22560 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
22561 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
22563 case NVPTX::BI__nvvm_fmin_f16:
22564 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID, E, *
this);
22565 case NVPTX::BI__nvvm_fmin_f16x2:
22566 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, *
this);
22567 case NVPTX::BI__nvvm_fmin_ftz_f16:
22568 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, *
this);
22569 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
22570 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, *
this);
22571 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
22572 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, *
this);
22573 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
22574 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
22576 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
22577 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
22579 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
22580 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
22581 BuiltinID, E, *
this);
22582 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
22583 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
22585 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
22586 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
22588 case NVPTX::BI__nvvm_fmin_nan_f16:
22589 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, *
this);
22590 case NVPTX::BI__nvvm_fmin_nan_f16x2:
22591 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, *
this);
22592 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
22593 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
22595 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
22596 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
22598 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
22599 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
22601 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
22602 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
22604 case NVPTX::BI__nvvm_ldg_h:
22605 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
22606 case NVPTX::BI__nvvm_ldg_h2:
22607 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
22608 case NVPTX::BI__nvvm_ldu_h:
22609 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
22610 case NVPTX::BI__nvvm_ldu_h2: {
22611 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
22613 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
22614 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
22615 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this, E,
22617 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
22618 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
22619 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this, E,
22621 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
22622 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
22623 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this, E,
22625 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
22626 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
22627 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this, E,
22629 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
22630 return Builder.CreateCall(
22631 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_x));
22632 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
22633 return Builder.CreateCall(
22634 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_y));
22635 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
22636 return Builder.CreateCall(
22637 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_z));
22638 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
22639 return Builder.CreateCall(
22640 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_w));
22641 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
22642 return Builder.CreateCall(
22643 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_x));
22644 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
22645 return Builder.CreateCall(
22646 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_y));
22647 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
22648 return Builder.CreateCall(
22649 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_z));
22650 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
22651 return Builder.CreateCall(
22652 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_w));
22653 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
22654 return Builder.CreateCall(
22655 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_x));
22656 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
22657 return Builder.CreateCall(
22658 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_y));
22659 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
22660 return Builder.CreateCall(
22661 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_z));
22662 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
22663 return Builder.CreateCall(
22664 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_w));
22665 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
22666 return Builder.CreateCall(
22667 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_x));
22668 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
22669 return Builder.CreateCall(
22670 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_y));
22671 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
22672 return Builder.CreateCall(
22673 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_z));
22674 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
22675 return Builder.CreateCall(
22676 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_w));
22677 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
22678 return Builder.CreateCall(
22679 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctarank));
22680 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
22681 return Builder.CreateCall(
22682 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctarank));
22683 case NVPTX::BI__nvvm_is_explicit_cluster:
22684 return Builder.CreateCall(
22685 CGM.getIntrinsic(Intrinsic::nvvm_is_explicit_cluster));
22686 case NVPTX::BI__nvvm_isspacep_shared_cluster:
22687 return Builder.CreateCall(
22688 CGM.getIntrinsic(Intrinsic::nvvm_isspacep_shared_cluster),
22689 EmitScalarExpr(E->
getArg(0)));
22690 case NVPTX::BI__nvvm_mapa:
22691 return Builder.CreateCall(
22692 CGM.getIntrinsic(Intrinsic::nvvm_mapa),
22693 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22694 case NVPTX::BI__nvvm_mapa_shared_cluster:
22695 return Builder.CreateCall(
22696 CGM.getIntrinsic(Intrinsic::nvvm_mapa_shared_cluster),
22697 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22698 case NVPTX::BI__nvvm_getctarank:
22699 return Builder.CreateCall(
22700 CGM.getIntrinsic(Intrinsic::nvvm_getctarank),
22701 EmitScalarExpr(E->
getArg(0)));
22702 case NVPTX::BI__nvvm_getctarank_shared_cluster:
22703 return Builder.CreateCall(
22704 CGM.getIntrinsic(Intrinsic::nvvm_getctarank_shared_cluster),
22705 EmitScalarExpr(E->
getArg(0)));
22706 case NVPTX::BI__nvvm_barrier_cluster_arrive:
22707 return Builder.CreateCall(
22708 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_arrive));
22709 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
22710 return Builder.CreateCall(
22711 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_arrive_relaxed));
22712 case NVPTX::BI__nvvm_barrier_cluster_wait:
22713 return Builder.CreateCall(
22714 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_wait));
22715 case NVPTX::BI__nvvm_fence_sc_cluster:
22716 return Builder.CreateCall(
22717 CGM.getIntrinsic(Intrinsic::nvvm_fence_sc_cluster));
22724 struct BuiltinAlignArgs {
22725 llvm::Value *Src =
nullptr;
22726 llvm::Type *SrcType =
nullptr;
22727 llvm::Value *Alignment =
nullptr;
22728 llvm::Value *Mask =
nullptr;
22729 llvm::IntegerType *IntType =
nullptr;
22737 SrcType = Src->getType();
22738 if (SrcType->isPointerTy()) {
22739 IntType = IntegerType::get(
22743 assert(SrcType->isIntegerTy());
22744 IntType = cast<llvm::IntegerType>(SrcType);
22747 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
22748 auto *One = llvm::ConstantInt::get(IntType, 1);
22749 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
22756 BuiltinAlignArgs Args(E, *
this);
22757 llvm::Value *SrcAddress = Args.Src;
22758 if (Args.SrcType->isPointerTy())
22760 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
22762 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
22763 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
22770 BuiltinAlignArgs Args(E, *
this);
22771 llvm::Value *SrcForMask = Args.Src;
22777 if (Args.Src->getType()->isPointerTy()) {
22778 if (getLangOpts().isSignedOverflowDefined())
22780 Builder.CreateGEP(Int8Ty, SrcForMask, Args.Mask,
"over_boundary");
22782 SrcForMask = EmitCheckedInBoundsGEP(Int8Ty, SrcForMask, Args.Mask,
22787 SrcForMask = Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
22791 llvm::Value *InvertedMask = Builder.CreateNot(Args.Mask,
"inverted_mask");
22792 llvm::Value *Result =
nullptr;
22793 if (Args.Src->getType()->isPointerTy()) {
22794 Result = Builder.CreateIntrinsic(
22795 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
22796 {SrcForMask, InvertedMask},
nullptr,
"aligned_result");
22798 Result = Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
22800 assert(Result->getType() == Args.SrcType);
22806 switch (BuiltinID) {
22807 case WebAssembly::BI__builtin_wasm_memory_size: {
22808 llvm::Type *ResultType = ConvertType(E->
getType());
22811 CGM.getIntrinsic(Intrinsic::wasm_memory_size, ResultType);
22812 return Builder.CreateCall(Callee, I);
22814 case WebAssembly::BI__builtin_wasm_memory_grow: {
22815 llvm::Type *ResultType = ConvertType(E->
getType());
22817 EmitScalarExpr(E->
getArg(1))};
22819 CGM.getIntrinsic(Intrinsic::wasm_memory_grow, ResultType);
22820 return Builder.CreateCall(Callee, Args);
22822 case WebAssembly::BI__builtin_wasm_tls_size: {
22823 llvm::Type *ResultType = ConvertType(E->
getType());
22824 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_size, ResultType);
22825 return Builder.CreateCall(Callee);
22827 case WebAssembly::BI__builtin_wasm_tls_align: {
22828 llvm::Type *ResultType = ConvertType(E->
getType());
22829 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_align, ResultType);
22830 return Builder.CreateCall(Callee);
22832 case WebAssembly::BI__builtin_wasm_tls_base: {
22833 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_base);
22834 return Builder.CreateCall(Callee);
22836 case WebAssembly::BI__builtin_wasm_throw: {
22839 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_throw);
22840 return Builder.CreateCall(Callee, {Tag, Obj});
22842 case WebAssembly::BI__builtin_wasm_rethrow: {
22843 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_rethrow);
22844 return Builder.CreateCall(Callee);
22846 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
22850 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_atomic_wait32);
22851 return Builder.CreateCall(Callee, {Addr,
Expected, Timeout});
22853 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
22857 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_atomic_wait64);
22858 return Builder.CreateCall(Callee, {Addr,
Expected, Timeout});
22860 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
22863 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_atomic_notify);
22864 return Builder.CreateCall(Callee, {Addr, Count});
22866 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
22867 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
22868 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
22869 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
22871 llvm::Type *ResT = ConvertType(E->
getType());
22873 CGM.getIntrinsic(Intrinsic::wasm_trunc_signed, {ResT, Src->
getType()});
22874 return Builder.CreateCall(Callee, {Src});
22876 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
22877 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
22878 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
22879 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
22881 llvm::Type *ResT = ConvertType(E->
getType());
22882 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_trunc_unsigned,
22884 return Builder.CreateCall(Callee, {Src});
22886 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
22887 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
22888 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
22889 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
22890 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
22892 llvm::Type *ResT = ConvertType(E->
getType());
22894 CGM.getIntrinsic(Intrinsic::fptosi_sat, {ResT, Src->
getType()});
22895 return Builder.CreateCall(Callee, {Src});
22897 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
22898 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
22899 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
22900 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
22901 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
22903 llvm::Type *ResT = ConvertType(E->
getType());
22905 CGM.getIntrinsic(Intrinsic::fptoui_sat, {ResT, Src->
getType()});
22906 return Builder.CreateCall(Callee, {Src});
22908 case WebAssembly::BI__builtin_wasm_min_f32:
22909 case WebAssembly::BI__builtin_wasm_min_f64:
22910 case WebAssembly::BI__builtin_wasm_min_f32x4:
22911 case WebAssembly::BI__builtin_wasm_min_f64x2: {
22915 CGM.getIntrinsic(Intrinsic::minimum, ConvertType(E->
getType()));
22916 return Builder.CreateCall(Callee, {LHS, RHS});
22918 case WebAssembly::BI__builtin_wasm_max_f32:
22919 case WebAssembly::BI__builtin_wasm_max_f64:
22920 case WebAssembly::BI__builtin_wasm_max_f32x4:
22921 case WebAssembly::BI__builtin_wasm_max_f64x2: {
22925 CGM.getIntrinsic(Intrinsic::maximum, ConvertType(E->
getType()));
22926 return Builder.CreateCall(Callee, {LHS, RHS});
22928 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
22929 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
22933 CGM.getIntrinsic(Intrinsic::wasm_pmin, ConvertType(E->
getType()));
22934 return Builder.CreateCall(Callee, {LHS, RHS});
22936 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
22937 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
22941 CGM.getIntrinsic(Intrinsic::wasm_pmax, ConvertType(E->
getType()));
22942 return Builder.CreateCall(Callee, {LHS, RHS});
22944 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
22945 case WebAssembly::BI__builtin_wasm_floor_f32x4:
22946 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
22947 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
22948 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
22949 case WebAssembly::BI__builtin_wasm_floor_f64x2:
22950 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
22951 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
22953 switch (BuiltinID) {
22954 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
22955 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
22958 case WebAssembly::BI__builtin_wasm_floor_f32x4:
22959 case WebAssembly::BI__builtin_wasm_floor_f64x2:
22962 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
22963 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
22966 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
22967 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
22971 llvm_unreachable(
"unexpected builtin ID");
22974 Function *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->
getType()));
22975 return Builder.CreateCall(Callee,
Value);
22977 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
22978 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_ref_null_extern);
22979 return Builder.CreateCall(Callee);
22981 case WebAssembly::BI__builtin_wasm_ref_null_func: {
22982 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_ref_null_func);
22983 return Builder.CreateCall(Callee);
22985 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
22988 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_swizzle);
22989 return Builder.CreateCall(Callee, {Src, Indices});
22991 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
22992 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
22993 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
22994 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
22995 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
22996 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
22997 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
22998 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8: {
23000 switch (BuiltinID) {
23001 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
23002 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
23003 IntNo = Intrinsic::sadd_sat;
23005 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
23006 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
23007 IntNo = Intrinsic::uadd_sat;
23009 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
23010 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
23011 IntNo = Intrinsic::wasm_sub_sat_signed;
23013 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
23014 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8:
23015 IntNo = Intrinsic::wasm_sub_sat_unsigned;
23018 llvm_unreachable(
"unexpected builtin ID");
23022 Function *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->
getType()));
23023 return Builder.CreateCall(Callee, {LHS, RHS});
23025 case WebAssembly::BI__builtin_wasm_abs_i8x16:
23026 case WebAssembly::BI__builtin_wasm_abs_i16x8:
23027 case WebAssembly::BI__builtin_wasm_abs_i32x4:
23028 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
23030 Value *
Neg = Builder.CreateNeg(Vec,
"neg");
23031 Constant *Zero = llvm::Constant::getNullValue(Vec->
getType());
23032 Value *ICmp = Builder.CreateICmpSLT(Vec, Zero,
"abscond");
23033 return Builder.CreateSelect(ICmp,
Neg, Vec,
"abs");
23035 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
23036 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
23037 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
23038 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
23039 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
23040 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
23041 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
23042 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
23043 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
23044 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
23045 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
23046 case WebAssembly::BI__builtin_wasm_max_u_i32x4: {
23050 switch (BuiltinID) {
23051 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
23052 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
23053 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
23054 ICmp = Builder.CreateICmpSLT(LHS, RHS);
23056 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
23057 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
23058 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
23059 ICmp = Builder.CreateICmpULT(LHS, RHS);
23061 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
23062 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
23063 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
23064 ICmp = Builder.CreateICmpSGT(LHS, RHS);
23066 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
23067 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
23068 case WebAssembly::BI__builtin_wasm_max_u_i32x4:
23069 ICmp = Builder.CreateICmpUGT(LHS, RHS);
23072 llvm_unreachable(
"unexpected builtin ID");
23074 return Builder.CreateSelect(ICmp, LHS, RHS);
23076 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
23077 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
23080 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_avgr_unsigned,
23082 return Builder.CreateCall(Callee, {LHS, RHS});
23084 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
23087 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_q15mulr_sat_signed);
23088 return Builder.CreateCall(Callee, {LHS, RHS});
23090 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
23091 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
23092 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
23093 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
23096 switch (BuiltinID) {
23097 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
23098 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
23099 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
23101 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
23102 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
23103 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
23106 llvm_unreachable(
"unexpected builtin ID");
23109 Function *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->
getType()));
23110 return Builder.CreateCall(Callee, Vec);
23112 case WebAssembly::BI__builtin_wasm_bitselect: {
23117 CGM.getIntrinsic(Intrinsic::wasm_bitselect, ConvertType(E->
getType()));
23118 return Builder.CreateCall(Callee, {V1, V2, C});
23120 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
23123 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_dot);
23124 return Builder.CreateCall(Callee, {LHS, RHS});
23126 case WebAssembly::BI__builtin_wasm_popcnt_i8x16: {
23129 CGM.getIntrinsic(Intrinsic::ctpop, ConvertType(E->
getType()));
23130 return Builder.CreateCall(Callee, {Vec});
23132 case WebAssembly::BI__builtin_wasm_any_true_v128:
23133 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
23134 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
23135 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
23136 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
23138 switch (BuiltinID) {
23139 case WebAssembly::BI__builtin_wasm_any_true_v128:
23140 IntNo = Intrinsic::wasm_anytrue;
23142 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
23143 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
23144 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
23145 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
23146 IntNo = Intrinsic::wasm_alltrue;
23149 llvm_unreachable(
"unexpected builtin ID");
23153 return Builder.CreateCall(Callee, {Vec});
23155 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
23156 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
23157 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
23158 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
23161 CGM.getIntrinsic(Intrinsic::wasm_bitmask, Vec->
getType());
23162 return Builder.CreateCall(Callee, {Vec});
23164 case WebAssembly::BI__builtin_wasm_abs_f32x4:
23165 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
23168 return Builder.CreateCall(Callee, {Vec});
23170 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
23171 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
23174 return Builder.CreateCall(Callee, {Vec});
23176 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
23177 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
23178 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
23179 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
23183 switch (BuiltinID) {
23184 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
23185 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
23186 IntNo = Intrinsic::wasm_narrow_signed;
23188 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
23189 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
23190 IntNo = Intrinsic::wasm_narrow_unsigned;
23193 llvm_unreachable(
"unexpected builtin ID");
23196 CGM.getIntrinsic(IntNo, {ConvertType(E->
getType()), Low->
getType()});
23197 return Builder.CreateCall(Callee, {Low, High});
23199 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
23200 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
23203 switch (BuiltinID) {
23204 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
23205 IntNo = Intrinsic::fptosi_sat;
23207 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
23208 IntNo = Intrinsic::fptoui_sat;
23211 llvm_unreachable(
"unexpected builtin ID");
23213 llvm::Type *SrcT = Vec->
getType();
23214 llvm::Type *TruncT = SrcT->getWithNewType(Builder.getInt32Ty());
23215 Function *Callee = CGM.getIntrinsic(IntNo, {TruncT, SrcT});
23216 Value *Trunc = Builder.CreateCall(Callee, Vec);
23217 Value *Splat = Constant::getNullValue(TruncT);
23218 return Builder.CreateShuffleVector(Trunc, Splat,
ArrayRef<int>{0, 1, 2, 3});
23220 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
23223 Ops[OpIdx++] = EmitScalarExpr(E->
getArg(0));
23224 Ops[OpIdx++] = EmitScalarExpr(E->
getArg(1));
23225 while (OpIdx < 18) {
23226 std::optional<llvm::APSInt> LaneConst =
23228 assert(LaneConst &&
"Constant arg isn't actually constant?");
23229 Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), *LaneConst);
23231 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_shuffle);
23232 return Builder.CreateCall(Callee, Ops);
23234 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
23235 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
23236 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
23237 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
23242 switch (BuiltinID) {
23243 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
23244 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
23245 IntNo = Intrinsic::wasm_relaxed_madd;
23247 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
23248 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
23249 IntNo = Intrinsic::wasm_relaxed_nmadd;
23252 llvm_unreachable(
"unexpected builtin ID");
23255 return Builder.CreateCall(Callee, {A, B, C});
23257 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
23258 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
23259 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
23260 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
23265 CGM.getIntrinsic(Intrinsic::wasm_relaxed_laneselect, A->
getType());
23266 return Builder.CreateCall(Callee, {A, B, C});
23268 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
23271 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_relaxed_swizzle);
23272 return Builder.CreateCall(Callee, {Src, Indices});
23274 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
23275 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
23276 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
23277 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
23281 switch (BuiltinID) {
23282 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
23283 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
23284 IntNo = Intrinsic::wasm_relaxed_min;
23286 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
23287 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
23288 IntNo = Intrinsic::wasm_relaxed_max;
23291 llvm_unreachable(
"unexpected builtin ID");
23294 return Builder.CreateCall(Callee, {LHS, RHS});
23296 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
23297 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
23298 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
23299 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
23302 switch (BuiltinID) {
23303 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
23304 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
23306 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
23307 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
23309 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
23310 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
23312 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
23313 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
23316 llvm_unreachable(
"unexpected builtin ID");
23318 Function *Callee = CGM.getIntrinsic(IntNo);
23319 return Builder.CreateCall(Callee, {Vec});
23321 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
23324 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_relaxed_q15mulr_signed);
23325 return Builder.CreateCall(Callee, {LHS, RHS});
23327 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
23331 CGM.getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed);
23332 return Builder.CreateCall(Callee, {LHS, RHS});
23334 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
23339 CGM.getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
23340 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
23342 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
23347 CGM.getIntrinsic(Intrinsic::wasm_relaxed_dot_bf16x8_add_f32);
23348 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
23350 case WebAssembly::BI__builtin_wasm_table_get: {
23352 Value *Table = EmitArrayToPointerDecay(E->
getArg(0)).emitRawPointer(*
this);
23356 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_get_externref);
23358 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_get_funcref);
23361 "Unexpected reference type for __builtin_wasm_table_get");
23362 return Builder.CreateCall(Callee, {Table, Index});
23364 case WebAssembly::BI__builtin_wasm_table_set: {
23366 Value *Table = EmitArrayToPointerDecay(E->
getArg(0)).emitRawPointer(*
this);
23371 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_set_externref);
23373 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_set_funcref);
23376 "Unexpected reference type for __builtin_wasm_table_set");
23377 return Builder.CreateCall(Callee, {Table, Index, Val});
23379 case WebAssembly::BI__builtin_wasm_table_size: {
23381 Value *
Value = EmitArrayToPointerDecay(E->
getArg(0)).emitRawPointer(*
this);
23382 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_size);
23383 return Builder.CreateCall(Callee,
Value);
23385 case WebAssembly::BI__builtin_wasm_table_grow: {
23387 Value *Table = EmitArrayToPointerDecay(E->
getArg(0)).emitRawPointer(*
this);
23393 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_grow_externref);
23395 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_fill_funcref);
23398 "Unexpected reference type for __builtin_wasm_table_grow");
23400 return Builder.CreateCall(Callee, {Table, Val, NElems});
23402 case WebAssembly::BI__builtin_wasm_table_fill: {
23404 Value *Table = EmitArrayToPointerDecay(E->
getArg(0)).emitRawPointer(*
this);
23411 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_fill_externref);
23413 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_fill_funcref);
23416 "Unexpected reference type for __builtin_wasm_table_fill");
23418 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
23420 case WebAssembly::BI__builtin_wasm_table_copy: {
23422 Value *TableX = EmitArrayToPointerDecay(E->
getArg(0)).emitRawPointer(*
this);
23423 Value *TableY = EmitArrayToPointerDecay(E->
getArg(1)).emitRawPointer(*
this);
23428 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_copy);
23430 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
23437 static std::pair<Intrinsic::ID, unsigned>
23440 unsigned BuiltinID;
23444 static Info Infos[] = {
23445 #define CUSTOM_BUILTIN_MAPPING(x,s) \
23446 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
23478 #include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
23479 #undef CUSTOM_BUILTIN_MAPPING
23482 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
23483 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
23486 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
23487 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
23488 return {Intrinsic::not_intrinsic, 0};
23490 return {F->IntrinsicID, F->VecLen};
23499 auto MakeCircOp = [
this, E](
unsigned IntID,
bool IsLoad) {
23503 llvm::Value *
Base = Builder.CreateLoad(BP);
23513 for (
unsigned i = 1, e = E->
getNumArgs(); i != e; ++i)
23514 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
23516 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
23519 llvm::Value *NewBase =
23520 IsLoad ? Builder.CreateExtractValue(Result, 1) : Result;
23521 llvm::Value *LV = EmitScalarExpr(E->
getArg(0));
23523 llvm::Value *RetVal =
23524 Builder.CreateAlignedStore(NewBase, LV, Dest.
getAlignment());
23526 RetVal = Builder.CreateExtractValue(Result, 0);
23533 auto MakeBrevLd = [
this, E](
unsigned IntID, llvm::Type *DestTy) {
23537 llvm::Value *BaseAddress = EmitScalarExpr(E->
getArg(0));
23549 llvm::Value *Result = Builder.CreateCall(
23550 CGM.getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
23553 llvm::Value *DestVal = Builder.CreateExtractValue(Result, 0);
23558 DestVal = Builder.CreateTrunc(DestVal, DestTy);
23560 Builder.CreateAlignedStore(DestVal, DestAddress, DestAddr.
getAlignment());
23562 return Builder.CreateExtractValue(Result, 1);
23565 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
23566 Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
23567 : Intrinsic::hexagon_V6_vandvrt;
23568 return Builder.CreateCall(CGM.getIntrinsic(
ID),
23569 {Vec, Builder.getInt32(-1)});
23571 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
23572 Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
23573 : Intrinsic::hexagon_V6_vandqrt;
23574 return Builder.CreateCall(CGM.getIntrinsic(
ID),
23575 {Pred, Builder.getInt32(-1)});
23578 switch (BuiltinID) {
23582 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
23583 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
23584 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
23585 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
23587 llvm::Type *VecType = ConvertType(E->
getArg(0)->
getType());
23590 llvm::Value *PredIn = V2Q(Builder.CreateLoad(PredAddr));
23591 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(
ID),
23592 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
23594 llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
23595 Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.
emitRawPointer(*
this),
23597 return Builder.CreateExtractValue(Result, 0);
23602 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
23603 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
23604 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
23605 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
23607 llvm::Type *VecType = ConvertType(E->
getArg(0)->
getType());
23610 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(
ID),
23611 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
23613 llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
23614 Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.
emitRawPointer(*
this),
23616 return Builder.CreateExtractValue(Result, 0);
23619 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
23620 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
23621 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
23622 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
23623 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
23624 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
23625 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
23626 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
23630 if (
auto *
Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
23631 if (
Cast->getCastKind() == CK_BitCast)
23632 PredOp =
Cast->getSubExpr();
23633 Ops.push_back(V2Q(EmitScalarExpr(PredOp)));
23635 for (
int i = 1, e = E->
getNumArgs(); i != e; ++i)
23636 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
23637 return Builder.CreateCall(CGM.getIntrinsic(
ID), Ops);
23640 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
23641 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
23642 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
23643 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
23644 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
23645 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
23646 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
23647 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
23648 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
23649 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
23650 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
23651 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
23652 return MakeCircOp(
ID,
true);
23653 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
23654 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
23655 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
23656 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
23657 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
23658 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
23659 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
23660 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
23661 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
23662 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
23663 return MakeCircOp(
ID,
false);
23664 case Hexagon::BI__builtin_brev_ldub:
23665 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr, Int8Ty);
23666 case Hexagon::BI__builtin_brev_ldb:
23667 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr, Int8Ty);
23668 case Hexagon::BI__builtin_brev_lduh:
23669 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr, Int16Ty);
23670 case Hexagon::BI__builtin_brev_ldh:
23671 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr, Int16Ty);
23672 case Hexagon::BI__builtin_brev_ldw:
23673 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr, Int32Ty);
23674 case Hexagon::BI__builtin_brev_ldd:
23675 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr, Int64Ty);
23685 StringRef AnnotStr =
"__builtin_intel_fpga_reg";
23688 Address DstAddr = ReturnValue.getValue();
23689 EmitAnyExprToMem(PtrArg, DstAddr, ArgType.
getQualifiers(),
true);
23691 EmitIntelFPGAFieldAnnotations(E->
getExprLoc(), DstAddr, AnnotStr);
23696 llvm::Value *
V = EmitScalarExpr(PtrArg);
23699 llvm::Type *OrigVType =
V->getType();
23700 if (!OrigVType->isIntegerTy()) {
23701 IntegerType *IntTy =
23702 Builder.getIntNTy(CGM.getDataLayout().getTypeSizeInBits(OrigVType));
23703 V = Builder.CreateBitOrPointerCast(
V, IntTy);
23705 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
23706 {
V->getType(), CGM.ConstGlobalsPtrTy});
23707 llvm::Value *AnnotatedV =
23708 EmitAnnotationCall(F,
V, AnnotStr, E->
getExprLoc());
23710 if (AnnotatedV->getType() != OrigVType) {
23711 AnnotatedV = Builder.CreateBitOrPointerCast(AnnotatedV, OrigVType);
23720 Value *PtrVal = EmitScalarExpr(PtrArg);
23724 Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
23725 {PtrVal->
getType(), CGM.ConstGlobalsPtrTy});
23727 llvm::raw_svector_ostream Out(AnnotStr);
23729 auto AddArgValue = [&E, &Ctx, &Out](
unsigned NumOfArg, StringRef StringToAdd,
23730 int DefaultValue =
INT_MIN) {
23731 std::optional<llvm::APSInt> IntVal =
23734 : APSInt::get(DefaultValue);
23735 assert(IntVal.has_value() &&
"Constant arg isn't actually constant?");
23736 Out <<
"{" << StringToAdd <<
":" <<
toString(*IntVal, 10) <<
"}";
23739 AddArgValue(1,
"params");
23740 AddArgValue(2,
"cache-size");
23747 AddArgValue(3,
"anchor-id", -1);
23748 AddArgValue(4,
"target-anchor", 0);
23749 AddArgValue(5,
"type", 0);
23750 AddArgValue(6,
"cycle", 0);
23752 llvm::Value *Ann = EmitAnnotationCall(F, PtrVal, AnnotStr,
SourceLocation());
23754 cast<CallBase>(Ann)->setDoesNotAccessMemory();
23767 llvm::Value *FnPtr, StringRef Name,
unsigned FDBuiltinID) {
23768 unsigned FPAccuracyIntrinsicID = 0;
23769 if (FDBuiltinID == 0) {
23770 FPAccuracyIntrinsicID =
23771 llvm::StringSwitch<unsigned>(Name)
23772 .Case(
"fadd", llvm::Intrinsic::fpbuiltin_fadd)
23773 .Case(
"fdiv", llvm::Intrinsic::fpbuiltin_fdiv)
23774 .Case(
"fmul", llvm::Intrinsic::fpbuiltin_fmul)
23775 .Case(
"fsub", llvm::Intrinsic::fpbuiltin_fsub)
23776 .Case(
"frem", llvm::Intrinsic::fpbuiltin_frem)
23777 .Case(
"sincos", llvm::Intrinsic::fpbuiltin_sincos)
23778 .Case(
"exp10", llvm::Intrinsic::fpbuiltin_exp10)
23779 .Case(
"rsqrt", llvm::Intrinsic::fpbuiltin_rsqrt)
23784 switch (FDBuiltinID) {
23790 case Builtin::BItan:
23791 case Builtin::BItanf:
23792 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_tan;
23794 case Builtin::BItanh:
23795 case Builtin::BItanhf:
23796 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_tanh;
23798 case Builtin::BIlog2:
23799 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_log2;
23801 case Builtin::BIlog1p:
23802 case Builtin::BIlog1pf:
23803 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_log1p;
23805 case Builtin::BIcos:
23806 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_cos;
23808 case Builtin::BIcosh:
23809 case Builtin::BIcoshf:
23810 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_cosh;
23812 case Builtin::BIacos:
23813 case Builtin::BIacosf:
23814 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_acos;
23816 case Builtin::BIacosh:
23817 case Builtin::BIacoshf:
23818 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_acosh;
23820 case Builtin::BIsin:
23821 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_sin;
23823 case Builtin::BIsinh:
23824 case Builtin::BIsinhf:
23825 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_sinh;
23827 case Builtin::BIasin:
23828 case Builtin::BIasinf:
23829 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_asin;
23831 case Builtin::BIasinh:
23832 case Builtin::BIasinhf:
23833 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_asinh;
23835 case Builtin::BIatan:
23836 case Builtin::BIatanf:
23837 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_atan;
23839 case Builtin::BIatanh:
23840 case Builtin::BIatanhf:
23841 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_atanh;
23843 case Builtin::BIatan2:
23844 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_atan2;
23846 case Builtin::BIerf:
23847 case Builtin::BIerff:
23848 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_erf;
23850 case Builtin::BIerfc:
23851 case Builtin::BIerfcf:
23852 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_erfc;
23854 case Builtin::BIexp:
23855 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_exp;
23857 case Builtin::BIexp2:
23858 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_exp2;
23860 case Builtin::BIexpm1:
23861 case Builtin::BIexpm1f:
23862 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_expm1;
23864 case Builtin::BIhypot:
23865 case Builtin::BIhypotf:
23866 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_hypot;
23868 case Builtin::BIldexp:
23869 case Builtin::BIldexpf:
23870 FPAccuracyIntrinsicID = Intrinsic::fpbuiltin_ldexp;
23874 if (!FPAccuracyIntrinsicID)
23882 llvm::Function *
Func =
23883 CGM.getIntrinsic(FPAccuracyIntrinsicID, IRArgs[0]->getType());
23885 FPAccuracyIntrinsicID);
23892 Value *PtrVal = EmitScalarExpr(PtrArg);
23893 auto &Ctx = CGM.getContext();
23896 Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
23897 {PtrVal->
getType(), CGM.ConstGlobalsPtrTy});
23900 llvm::raw_svector_ostream Out(AnnotStr);
23904 for (
unsigned I = 1, N = E->
getNumArgs(); I <= N / 2; I++) {
23906 const StringLiteral *Str = dyn_cast<const StringLiteral>(Arg);
23908 if (!Str && Arg->
EvaluateAsRValue(Result, Ctx) && Result.Val.isLValue()) {
23909 const auto *LVE = Result.Val.getLValueBase().dyn_cast<
const Expr *>();
23910 Str = dyn_cast<const StringLiteral>(LVE);
23912 assert(Str &&
"Constant parameter string is invalid?");
23915 assert(IntVal.has_value() &&
23916 "Constant integer arg isn't actually constant?");
23918 Properties.push_back(
23923 EmitSYCLAnnotationCall(F, PtrVal, E->
getExprLoc(), Properties);
23929 assert((BuiltinID == Builtin::BI__builtin_intel_sycl_alloca ||
23930 BuiltinID == Builtin::BI__builtin_intel_sycl_alloca_with_align) &&
23931 "Unexpected builtin");
23933 bool IsAlignedAlloca =
23934 BuiltinID == Builtin::BI__builtin_intel_sycl_alloca_with_align;
23936 constexpr
unsigned InvalidIndex = -1;
23937 constexpr
unsigned ElementTypeIndex = 0;
23938 const unsigned AlignmentIndex = IsAlignedAlloca ? 1 : InvalidIndex;
23939 const unsigned SpecNameIndex = IsAlignedAlloca ? 2 : 1;
23940 const unsigned DecorateAddressIndex = IsAlignedAlloca ? 3 : 2;
23943 assert(FD &&
"Expecting direct call to builtin");
23949 assert(TAL &&
"Expecting template argument list");
23955 llvm::Value *UID = EmitScalarExpr(
23959 llvm::Value *SpecConstPtr =
23965 llvm::Value *RTBufferPtr = llvm::ConstantPointerNull::get(
23966 cast<llvm::PointerType>(SpecConstPtr->getType()));
23970 llvm::Type *Ty = CGM.getTypes().ConvertTypeForMem(AllocaType);
23971 unsigned AllocaAS = CGM.getDataLayout().getAllocaAddrSpace();
23972 llvm::Type *AllocaTy = llvm::PointerType::get(Builder.getContext(), AllocaAS);
23974 llvm::Constant *EltTyConst = llvm::Constant::getNullValue(Ty);
23976 llvm::Constant *Align = Builder.getInt64(
23979 : getContext().getTypeAlignInChars(AllocaType).getAsAlign().value());
23981 llvm::Value *Allocation = [&]() {
23985 IRBuilderBase::InsertPointGuard IPG(Builder);
23986 Builder.SetInsertPoint(AllocaInsertPt);
23987 llvm::CallInst *CI = Builder.CreateIntrinsic(
23988 AllocaTy, Intrinsic::sycl_alloca,
23989 {UID, SpecConstPtr, RTBufferPtr, EltTyConst, Align},
nullptr,
"alloca");
23992 llvm::Function *F = CI->getCalledFunction();
23993 constexpr llvm::StringLiteral MDName =
"sycl_used_aspects";
23994 if (!F->getMetadata(MDName)) {
23995 auto *AspectAttr = FD->
getAttr<SYCLUsesAspectsAttr>();
23996 assert(AspectAttr && AspectAttr->aspects_size() == 1 &&
23997 "Expecting a single aspect");
23999 (*AspectAttr->aspects_begin())->EvaluateKnownConstInt(getContext());
24000 llvm::Type *I32Ty = Builder.getInt32Ty();
24001 llvm::Constant *C = llvm::Constant::getIntegerValue(I32Ty, AspectInt);
24002 llvm::Metadata *AspectMD = llvm::ConstantAsMetadata::get(C);
24003 F->setMetadata(MDName, llvm::MDNode::get(Builder.getContext(), AspectMD));
24010 constexpr
int NoDecorated = 0;
24014 if (Decorated == NoDecorated) {
24015 IRBuilderBase::InsertPointGuard IPG(Builder);
24016 Builder.SetInsertPoint(getPostAllocaInsertPoint());
24017 unsigned DestAddrSpace =
24019 llvm::PointerType *DestTy =
24020 llvm::PointerType::get(Builder.getContext(), DestAddrSpace);
24021 Allocation = Builder.CreateAddrSpaceCast(Allocation, DestTy);
24025 if (ReturnValue.isNull())
24029 Builder.CreateStore(Allocation, ReturnValue.getValue());
24037 llvm::Type *ResultType = ConvertType(E->
getType());
24040 unsigned ICEArguments = 0;
24042 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
24048 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
24049 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
24050 ICEArguments = 1 << 1;
24055 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
24056 ICEArguments |= (1 << 1);
24057 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
24058 ICEArguments |= (1 << 2);
24060 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
24064 llvm::Value *AggValue = Builder.CreateLoad(L.
getAddress(*
this));
24065 Ops.push_back(AggValue);
24068 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E));
24075 constexpr
unsigned RVV_VTA = 0x1;
24076 constexpr
unsigned RVV_VMA = 0x2;
24077 int PolicyAttrs = 0;
24078 bool IsMasked =
false;
24082 switch (BuiltinID) {
24083 default: llvm_unreachable(
"unexpected builtin ID");
24084 case RISCV::BI__builtin_riscv_orc_b_32:
24085 case RISCV::BI__builtin_riscv_orc_b_64:
24086 case RISCV::BI__builtin_riscv_clz_32:
24087 case RISCV::BI__builtin_riscv_clz_64:
24088 case RISCV::BI__builtin_riscv_ctz_32:
24089 case RISCV::BI__builtin_riscv_ctz_64:
24090 case RISCV::BI__builtin_riscv_clmul_32:
24091 case RISCV::BI__builtin_riscv_clmul_64:
24092 case RISCV::BI__builtin_riscv_clmulh_32:
24093 case RISCV::BI__builtin_riscv_clmulh_64:
24094 case RISCV::BI__builtin_riscv_clmulr_32:
24095 case RISCV::BI__builtin_riscv_clmulr_64:
24096 case RISCV::BI__builtin_riscv_xperm4_32:
24097 case RISCV::BI__builtin_riscv_xperm4_64:
24098 case RISCV::BI__builtin_riscv_xperm8_32:
24099 case RISCV::BI__builtin_riscv_xperm8_64:
24100 case RISCV::BI__builtin_riscv_brev8_32:
24101 case RISCV::BI__builtin_riscv_brev8_64:
24102 case RISCV::BI__builtin_riscv_zip_32:
24103 case RISCV::BI__builtin_riscv_unzip_32: {
24104 switch (BuiltinID) {
24105 default: llvm_unreachable(
"unexpected builtin ID");
24107 case RISCV::BI__builtin_riscv_orc_b_32:
24108 case RISCV::BI__builtin_riscv_orc_b_64:
24109 ID = Intrinsic::riscv_orc_b;
24111 case RISCV::BI__builtin_riscv_clz_32:
24112 case RISCV::BI__builtin_riscv_clz_64: {
24113 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
24114 Value *Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
24115 if (Result->getType() != ResultType)
24116 Result = Builder.CreateIntCast(Result, ResultType,
true,
24120 case RISCV::BI__builtin_riscv_ctz_32:
24121 case RISCV::BI__builtin_riscv_ctz_64: {
24122 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
24123 Value *Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
24124 if (Result->getType() != ResultType)
24125 Result = Builder.CreateIntCast(Result, ResultType,
true,
24131 case RISCV::BI__builtin_riscv_clmul_32:
24132 case RISCV::BI__builtin_riscv_clmul_64:
24133 ID = Intrinsic::riscv_clmul;
24135 case RISCV::BI__builtin_riscv_clmulh_32:
24136 case RISCV::BI__builtin_riscv_clmulh_64:
24137 ID = Intrinsic::riscv_clmulh;
24139 case RISCV::BI__builtin_riscv_clmulr_32:
24140 case RISCV::BI__builtin_riscv_clmulr_64:
24141 ID = Intrinsic::riscv_clmulr;
24145 case RISCV::BI__builtin_riscv_xperm8_32:
24146 case RISCV::BI__builtin_riscv_xperm8_64:
24147 ID = Intrinsic::riscv_xperm8;
24149 case RISCV::BI__builtin_riscv_xperm4_32:
24150 case RISCV::BI__builtin_riscv_xperm4_64:
24151 ID = Intrinsic::riscv_xperm4;
24155 case RISCV::BI__builtin_riscv_brev8_32:
24156 case RISCV::BI__builtin_riscv_brev8_64:
24157 ID = Intrinsic::riscv_brev8;
24159 case RISCV::BI__builtin_riscv_zip_32:
24160 ID = Intrinsic::riscv_zip;
24162 case RISCV::BI__builtin_riscv_unzip_32:
24163 ID = Intrinsic::riscv_unzip;
24167 IntrinsicTypes = {ResultType};
24174 case RISCV::BI__builtin_riscv_sha256sig0:
24175 ID = Intrinsic::riscv_sha256sig0;
24177 case RISCV::BI__builtin_riscv_sha256sig1:
24178 ID = Intrinsic::riscv_sha256sig1;
24180 case RISCV::BI__builtin_riscv_sha256sum0:
24181 ID = Intrinsic::riscv_sha256sum0;
24183 case RISCV::BI__builtin_riscv_sha256sum1:
24184 ID = Intrinsic::riscv_sha256sum1;
24188 case RISCV::BI__builtin_riscv_sm4ks:
24189 ID = Intrinsic::riscv_sm4ks;
24191 case RISCV::BI__builtin_riscv_sm4ed:
24192 ID = Intrinsic::riscv_sm4ed;
24196 case RISCV::BI__builtin_riscv_sm3p0:
24197 ID = Intrinsic::riscv_sm3p0;
24199 case RISCV::BI__builtin_riscv_sm3p1:
24200 ID = Intrinsic::riscv_sm3p1;
24204 case RISCV::BI__builtin_riscv_ntl_load: {
24205 llvm::Type *ResTy = ConvertType(E->
getType());
24206 unsigned DomainVal = 5;
24207 if (Ops.size() == 2)
24208 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
24210 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
24212 llvm::ConstantAsMetadata::get(Builder.getInt32(DomainVal)));
24213 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
24214 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
24217 if(ResTy->isScalableTy()) {
24218 const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
24219 llvm::Type *ScalarTy = ResTy->getScalarType();
24220 Width = ScalarTy->getPrimitiveSizeInBits() *
24221 SVTy->getElementCount().getKnownMinValue();
24223 Width = ResTy->getPrimitiveSizeInBits();
24224 LoadInst *
Load = Builder.CreateLoad(
24227 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
24228 Load->setMetadata(CGM.getModule().getMDKindID(
"riscv-nontemporal-domain"),
24233 case RISCV::BI__builtin_riscv_ntl_store: {
24234 unsigned DomainVal = 5;
24235 if (Ops.size() == 3)
24236 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
24238 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
24240 llvm::ConstantAsMetadata::get(Builder.getInt32(DomainVal)));
24241 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
24242 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
24244 StoreInst *
Store = Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
24245 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
24246 Store->setMetadata(CGM.getModule().getMDKindID(
"riscv-nontemporal-domain"),
24253 #include "clang/Basic/riscv_vector_builtin_cg.inc"
24255 #include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
24258 assert(
ID != Intrinsic::not_intrinsic);
24260 llvm::Function *F = CGM.getIntrinsic(
ID, IntrinsicTypes);
24261 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool isExecHi)
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, unsigned FPAccuracyIntrinsicID=Intrinsic::not_intrinsic)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
#define X86_CPU_SUBTYPE(ENUM, STR)
#define LD_CASES_AS_SCOPES(ORDER)
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
#define ST_VOLATILE_CASES(ADDR_SPACE)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
#define MMA_VARIANTS_B1_AND(geom, type)
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static bool hasFuncNameRequestedFPAccuracy(StringRef Name, const LangOptions &LangOpts)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
#define MMA_INTR(geom_op_type, layout)
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static bool hasAccuracyRequirement(CodeGenFunction &CGF, StringRef Name)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
static Value * EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
#define MUTATE_LDBL(func)
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static unsigned CountCountedByAttrs(const RecordDecl *RD)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
constexpr unsigned SVEBitsPerBlock
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define X86_VENDOR(ENUM, STRING)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
#define NEONMAP0(NameBase)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static Function * getIntrinsic(CodeGenFunction &CGF, llvm::Value *Src0, unsigned FPIntrinsicID, unsigned IntrinsicID, bool HasAccuracyRequirement)
#define PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD)
#define MMA_SATF_VARIANTS(geom, type)
static CallInst * CreateBuiltinCallWithAttr(CodeGenFunction &CGF, StringRef Name, llvm::Function *FPBuiltinF, ArrayRef< Value * > Args, unsigned ID)
static Function * emitMaybeIntrinsic(CodeGenFunction &CGF, const CallExpr *E, unsigned FPAccuracyIntrinsicID, unsigned IntrinsicID, llvm::Value *Src0, StringRef &Name)
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, unsigned FPAccuracyIntrinsicID=Intrinsic::not_intrinsic)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
#define LD_VOLATILE_CASES(ADDR_SPACE)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define ST_CASES_AS_SCOPES(ORDER)
#define CUSTOM_BUILTIN_MAPPING(x, s)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
#define INTRINSIC_WITH_CC(NAME)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E, bool SanitizeOverflow)
SpecialRegisterAccessKind
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static bool NEONSIMDIntrinsicsProvenSorted
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
#define MMA_LDST(n, geom_op_type)
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, const FunctionDecl *FD)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
#define X86_CPU_TYPE(ENUM, STR)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
Intrinsic::ID getDotProductIntrinsic(QualType QT, int elementCount)
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo, KnownNonNull_t IsKnownNonNull, CodeGenFunction &CGF)
#define ALIAS(NAME, TOK, FLAGS)
llvm::MachO::Record Record
static std::string getName(const CallEvent &Call)
Enumerates target-specific builtins in their own namespaces within namespace clang.
Defines the clang::TargetOptions class.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ long long abs(long long __n)
__DEVICE__ double powi(double __a, int __b)
__DEVICE__ int min(int __a, int __b)
__DEVICE__ unsigned int umin(unsigned int __a, unsigned int __b)
__DEVICE__ unsigned int umax(unsigned int __a, unsigned int __b)
__DEVICE__ double exp10(double __a)
__DEVICE__ int max(int __a, int __b)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
unsigned getIntWidth(QualType T) const
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
QualType getElementType() const
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
bool hasStoredFPFeatures() const
SourceLocation getBeginLoc() const LLVM_READONLY
FPOptionsOverride getFPFeatures() const
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return null.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
CharUnits - This is an opaque type for sizes expressed in character units.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
llvm::PointerType * getType() const
Return the type of the pointer value.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
Address getAddress() const
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
CGFunctionInfo - Class to encapsulate the information about a function definition.
virtual llvm::Value * getPipeElemAlign(const Expr *PipeArg)
virtual llvm::Value * getPipeElemSize(const Expr *PipeArg)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
Class to manage the BuiltinID for the current builtin expression during processing in EmitBuiltinExpr...
RAII object to set/unset CodeGenFunction::IsSanitizerScope.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
unsigned getCurrentBuiltinID() const
llvm::Value * EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::Value * FormSVEBuiltinResult(llvm::Value *Call)
FormSVEBuiltinResult - Returns the struct of scalable vectors as a wider vector.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerMask >> Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertType(QualType T)
BuiltinCheckKind
Specifies which type of sanitizer check to apply when handling a particular builtin.
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
Emits a call or invoke instruction to the given runtime function.
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
llvm::LLVMContext & getLLVMContext()
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
RValue EmitIntelSYCLAllocaBuiltin(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
TypeCheckKind
Situations in which we might emit a check for the suitability of a pointer or glvalue.
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Constant * EmitCheckTypeDescriptor(QualType T)
Emit a description of a type in a format suitable for passing to a runtime sanitizer handler.
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
ASTContext & getContext() const
RValue EmitIntelFPGARegBuiltin(const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
llvm::CallBase * addControlledConvergenceToken(llvm::CallBase *Input)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
const TargetInfo & getTarget() const
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
llvm::CallInst * MaybeEmitFPBuiltinofFD(llvm::FunctionType *IRFuncTy, const SmallVectorImpl< llvm::Value * > &IRArgs, llvm::Value *FnPtr, StringRef Name, unsigned FDBuiltinID)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
RValue EmitIntelSYCLPtrAnnotationBuiltin(const CallExpr *E)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID)
Create a basic block that will call the trap intrinsic, and emit a conditional branch to it,...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitIntelFPGAMemBuiltin(const CallExpr *E)
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
static Destroyer destroyARCStrongPrecise
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
const FieldDecl * FindFlexibleArrayMemberField(ASTContext &Ctx, const RecordDecl *RD, StringRef Name, uint64_t &Offset)
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
Given a number of pointers, inform the optimizer that they're being intrinsically used up until this ...
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
const LangOptions & getLangOpts() const
This class organizes the cross-function state that is used while generating LLVM code.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
void getFPAccuracyFuncAttributes(StringRef Name, llvm::AttributeList &AttrList, llvm::Metadata *&MDs, unsigned ID, const llvm::Type *FuncType)
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
llvm::Module & getModule() const
llvm::LLVMContext & getLLVMContext()
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
DiagnosticsEngine & getDiags() const
ASTContext & getContext() const
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
const CodeGenOptions & getCodeGenOpts() const
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=std::nullopt)
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
Address getAddress(CodeGenFunction &CGF) const
void setNontemporal(bool Value)
Information used when generating a structured loop.
const LoopInfo * getParent() const
Returns the first outer loop containing this loop if any, nullptr otherwise.
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue getIgnored()
static RValue get(llvm::Value *V)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
An abstract representation of an aligned address.
llvm::Value * getPointer() const
static RawAddress invalid()
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
RecordDecl * getOuterLexicalRecordContext()
Retrieve the outermost lexically enclosing record context.
A reference to a declared variable, function, enum, etc.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
static bool isFlexibleArrayMemberLike(ASTContext &Context, const Decl *D, QualType Ty, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution)
Whether it resembles a flexible array member.
DeclContext * getDeclContext()
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
const TemplateArgumentList * getTemplateSpecializationArgs() const
Retrieve the template arguments used to produce this function template specialization from the primar...
const ParmVarDecl * getParamDecl(unsigned i) const
Represents a prototype with parameter type info, e.g.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StrictFlexArraysLevelKind
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
std::string FPAccuracyVal
FPAccuracyFuncMapTy FPAccuracyFuncMap
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
std::string getNameAsString() const
Get a human-readable name for the declaration, even if it is one of the special kinds of names (C++ c...
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
A C++ nested-name-specifier augmented with source location information.
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Represents a struct/union/class.
field_range fields() const
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
static SYCLUniqueStableIdExpr * Create(const ASTContext &Ctx, SourceLocation OpLoc, SourceLocation LParen, SourceLocation RParen, Expr *E)
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
StringLiteral - This represents a string literal expression, e.g.
StringRef getString() const
Exposes information about the current target.
unsigned getMaxOpenCLWorkGroupSize() const
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
TargetOptions & getTargetOpts() const
Retrieve the target options.
llvm::CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
A template argument list.
const TemplateArgument & get(unsigned Idx) const
Retrieve the template argument at a given index.
QualType getAsType() const
Retrieve the type for a type template argument.
llvm::APSInt getAsIntegral() const
Retrieve the template argument as an integral value.
ValueDecl * getAsDecl() const
Retrieve the declaration for a declaration non-type template argument.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isBooleanType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isComplexType() const
isComplexType() does not include complex integers (a GCC extension).
bool isCountAttributedType() const
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
bool hasUnsignedIntegerRepresentation() const
Determine whether this type has an unsigned integer representation of some sort, e....
bool isBitIntType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isObjCObjectPointerType() const
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a GCC generic vector type.
unsigned getNumElements() const
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask All
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
@ System
Like Angled, but marks system directories.
bool Sub(InterpState &S, CodePtr OpPC)
bool Dup(InterpState &S, CodePtr OpPC)
bool Ret(InterpState &S, CodePtr &PC, APValue &Result)
bool Mul(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
bool Add(InterpState &S, CodePtr OpPC)
std::string toString(const til::SExpr *E)
AccessKind
This enum distinguishes between different ways to access (read or write) a variable.
The JSON file list parser is used to communicate input to InstallAPI.
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Asm
Assembly: we accept this only so that we can preprocess it.
LangAS
Defines the address space values used by the address space qualifier of QualType.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
const FunctionProtoType * T
CallingConv
CallingConv - Specifies the calling convention that a function uses.
@ Other
Other implicit parameter.
@ HiddenVisibility
Objects with "hidden" visibility are not seen by the dynamic linker.
half clamp(half, half, half)
Diagnostic wrappers for TextAPI types for error reporting.
char __ovld __cnfn rotate(char, char)
For each element in v, the bits are shifted left by the number of bits given by the corresponding ele...
void __ovld prefetch(const __global char *, size_t)
Prefetch num_elements * sizeof(gentype) bytes into the global cache.
#define ptrauth_strip(__value, __key)
llvm::IntegerType * Int64Ty
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * UnqualPtrTy
llvm::PointerType * AllocaInt8PtrTy
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
void clear(SanitizerMask K=SanitizerKind::All)
Disable the sanitizers specified in K.
void set(SanitizerMask K, bool Value)
Enable or disable a certain (single) sanitizer.
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)